camelsch 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- camelsch-0.1.0/PKG-INFO +141 -0
- camelsch-0.1.0/README.md +119 -0
- camelsch-0.1.0/pyproject.toml +66 -0
- camelsch-0.1.0/src/camelsch/__init__.py +30 -0
- camelsch-0.1.0/src/camelsch/__main__.py +5 -0
- camelsch-0.1.0/src/camelsch/attributes.py +84 -0
- camelsch-0.1.0/src/camelsch/cli.py +357 -0
- camelsch-0.1.0/src/camelsch/download.py +110 -0
- camelsch-0.1.0/src/camelsch/export.py +85 -0
- camelsch-0.1.0/src/camelsch/io.py +87 -0
- camelsch-0.1.0/src/camelsch/py.typed +0 -0
- camelsch-0.1.0/src/camelsch/timeseries.py +164 -0
camelsch-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: camelsch
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: CLI tool for extracting and working with CAMELS-CH hydrological data
|
|
5
|
+
Author: Beatrice Marti
|
|
6
|
+
Author-email: Beatrice Marti <mabesa@users.noreply.github.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Requires-Dist: pandas>=2.0
|
|
9
|
+
Requires-Dist: typer>=0.9.0
|
|
10
|
+
Requires-Dist: rich>=13.0.0
|
|
11
|
+
Requires-Dist: pyarrow>=14.0.0
|
|
12
|
+
Requires-Dist: mypy>=1.10 ; extra == 'dev'
|
|
13
|
+
Requires-Dist: pandas-stubs>=2.0 ; extra == 'dev'
|
|
14
|
+
Requires-Dist: pytest>=9.0.2 ; extra == 'dev'
|
|
15
|
+
Requires-Dist: ruff>=0.15.2 ; extra == 'dev'
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Project-URL: Homepage, https://github.com/hydrosolutions/camelsch
|
|
18
|
+
Project-URL: Repository, https://github.com/hydrosolutions/camelsch
|
|
19
|
+
Project-URL: Issues, https://github.com/hydrosolutions/camelsch/issues
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# camelsch
|
|
24
|
+
|
|
25
|
+
[](https://github.com/hydrosolutions/camelsch/actions/workflows/ci.yml)
|
|
26
|
+
[](https://www.python.org)
|
|
27
|
+
[](https://mybinder.org/v2/gh/hydrosolutions/camelsch/HEAD?labpath=demo.ipynb)
|
|
28
|
+
|
|
29
|
+
CLI tool for downloading, exploring, and extracting data from the [CAMELS-CH](https://zenodo.org/records/15025258) hydrological dataset (331 Swiss basins, 1981-2020).
|
|
30
|
+
|
|
31
|
+
## Installation
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
pip install camelsch
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
This also works inside conda/mamba environments — just activate yours first.
|
|
38
|
+
|
|
39
|
+
If you use [uv](https://docs.astral.sh/uv/) for project management:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
uv add camelsch
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Workflow
|
|
46
|
+
|
|
47
|
+
camelsch follows a **download-then-query** workflow:
|
|
48
|
+
|
|
49
|
+
1. **Download** the CAMELS-CH dataset once from Zenodo (~1.5 GB).
|
|
50
|
+
2. **Explore** the dataset — list basins, check available variables, view summaries.
|
|
51
|
+
3. **Extract** the data you need — filter by basin, variable, and date range, then export to CSV or Parquet.
|
|
52
|
+
|
|
53
|
+
All commands read from a local data directory (default `./data/CAMELS_CH`). You can change it with `--data-dir` or the `CAMELSCH_DATA_DIR` environment variable.
|
|
54
|
+
|
|
55
|
+
## Quick start
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
# Step 1: Download the dataset from Zenodo (~1.5 GB)
|
|
59
|
+
camelsch download
|
|
60
|
+
|
|
61
|
+
# Step 2: Explore — show dataset summary
|
|
62
|
+
camelsch info
|
|
63
|
+
|
|
64
|
+
# List all basin IDs
|
|
65
|
+
camelsch basins
|
|
66
|
+
camelsch basins --format json
|
|
67
|
+
camelsch basins --format csv --attrs area,p_mean
|
|
68
|
+
|
|
69
|
+
# Step 3: Extract data
|
|
70
|
+
# Static catchment attributes
|
|
71
|
+
camelsch attributes --basins 2004,2007 --output attrs.csv
|
|
72
|
+
|
|
73
|
+
# Time series with filtering
|
|
74
|
+
camelsch timeseries --basins 2004 --vars precipitation,discharge_spec \
|
|
75
|
+
--start 1990-01-01 --end 2000-12-31 --output ts.csv
|
|
76
|
+
|
|
77
|
+
# Batch export to Parquet (with optional attribute merge)
|
|
78
|
+
camelsch export --include-attrs --output camels_ch.parquet
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## Python API
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
import camelsch
|
|
85
|
+
|
|
86
|
+
# Load static attributes for specific basins
|
|
87
|
+
attrs = camelsch.load_attributes("./data/CAMELS_CH", basin_ids=["2004", "2007"])
|
|
88
|
+
|
|
89
|
+
# Load time series with filtering
|
|
90
|
+
data = camelsch.load_timeseries(
|
|
91
|
+
"./data/CAMELS_CH",
|
|
92
|
+
basin_ids=["2004"],
|
|
93
|
+
variables=["precipitation", "discharge_spec"],
|
|
94
|
+
start_date="1990-01-01",
|
|
95
|
+
end_date="2000-12-31",
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
# List available basins and variables
|
|
99
|
+
basins = camelsch.list_basins("./data/CAMELS_CH")
|
|
100
|
+
variables = camelsch.list_variables("./data/CAMELS_CH")
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## Data reference
|
|
104
|
+
|
|
105
|
+
CAMELS-CH provides daily hydrometeorological time series and static catchment attributes for 331 Swiss basins (1981-2020). Key variables include:
|
|
106
|
+
|
|
107
|
+
| Variable | Description |
|
|
108
|
+
|---|---|
|
|
109
|
+
| `precipitation` | Basin-mean daily precipitation (mm/d) |
|
|
110
|
+
| `temperature` | Basin-mean daily temperature (deg C) |
|
|
111
|
+
| `discharge_spec` | Observed specific discharge (mm/d) |
|
|
112
|
+
| `discharge_spec_sim` | Simulated specific discharge (mm/d) |
|
|
113
|
+
| `pet` / `pet_sim` | Potential evapotranspiration (mm/d) |
|
|
114
|
+
| `et` / `et_sim` | Actual evapotranspiration (mm/d) |
|
|
115
|
+
| `snow_water_eq_sim` | Simulated snow water equivalent (mm) |
|
|
116
|
+
|
|
117
|
+
Static attributes cover topography (`area`, `elev_mean`), climate (`p_mean`, `t_mean`), land cover, soil, and geology.
|
|
118
|
+
|
|
119
|
+
**Full dataset**: [CAMELS-CH on Zenodo](https://zenodo.org/records/15025258)
|
|
120
|
+
**Reference paper**: Hoege et al. (2023), *CAMELS-CH: hydro-meteorological time series and landscape attributes for 331 catchments in hydrologic Switzerland*, Earth Syst. Sci. Data.
|
|
121
|
+
|
|
122
|
+
## For developers
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
# Install dependencies
|
|
126
|
+
uv sync
|
|
127
|
+
|
|
128
|
+
# Run tests
|
|
129
|
+
uv run pytest
|
|
130
|
+
|
|
131
|
+
# Lint and format
|
|
132
|
+
uv run ruff check src/ tests/
|
|
133
|
+
uv run ruff format src/ tests/
|
|
134
|
+
|
|
135
|
+
# Type check
|
|
136
|
+
uv run mypy src/
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
## License
|
|
140
|
+
|
|
141
|
+
MIT
|
camelsch-0.1.0/README.md
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# camelsch
|
|
2
|
+
|
|
3
|
+
[](https://github.com/hydrosolutions/camelsch/actions/workflows/ci.yml)
|
|
4
|
+
[](https://www.python.org)
|
|
5
|
+
[](https://mybinder.org/v2/gh/hydrosolutions/camelsch/HEAD?labpath=demo.ipynb)
|
|
6
|
+
|
|
7
|
+
CLI tool for downloading, exploring, and extracting data from the [CAMELS-CH](https://zenodo.org/records/15025258) hydrological dataset (331 Swiss basins, 1981-2020).
|
|
8
|
+
|
|
9
|
+
## Installation
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install camelsch
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
This also works inside conda/mamba environments — just activate yours first.
|
|
16
|
+
|
|
17
|
+
If you use [uv](https://docs.astral.sh/uv/) for project management:
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
uv add camelsch
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
## Workflow
|
|
24
|
+
|
|
25
|
+
camelsch follows a **download-then-query** workflow:
|
|
26
|
+
|
|
27
|
+
1. **Download** the CAMELS-CH dataset once from Zenodo (~1.5 GB).
|
|
28
|
+
2. **Explore** the dataset — list basins, check available variables, view summaries.
|
|
29
|
+
3. **Extract** the data you need — filter by basin, variable, and date range, then export to CSV or Parquet.
|
|
30
|
+
|
|
31
|
+
All commands read from a local data directory (default `./data/CAMELS_CH`). You can change it with `--data-dir` or the `CAMELSCH_DATA_DIR` environment variable.
|
|
32
|
+
|
|
33
|
+
## Quick start
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
# Step 1: Download the dataset from Zenodo (~1.5 GB)
|
|
37
|
+
camelsch download
|
|
38
|
+
|
|
39
|
+
# Step 2: Explore — show dataset summary
|
|
40
|
+
camelsch info
|
|
41
|
+
|
|
42
|
+
# List all basin IDs
|
|
43
|
+
camelsch basins
|
|
44
|
+
camelsch basins --format json
|
|
45
|
+
camelsch basins --format csv --attrs area,p_mean
|
|
46
|
+
|
|
47
|
+
# Step 3: Extract data
|
|
48
|
+
# Static catchment attributes
|
|
49
|
+
camelsch attributes --basins 2004,2007 --output attrs.csv
|
|
50
|
+
|
|
51
|
+
# Time series with filtering
|
|
52
|
+
camelsch timeseries --basins 2004 --vars precipitation,discharge_spec \
|
|
53
|
+
--start 1990-01-01 --end 2000-12-31 --output ts.csv
|
|
54
|
+
|
|
55
|
+
# Batch export to Parquet (with optional attribute merge)
|
|
56
|
+
camelsch export --include-attrs --output camels_ch.parquet
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Python API
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
import camelsch
|
|
63
|
+
|
|
64
|
+
# Load static attributes for specific basins
|
|
65
|
+
attrs = camelsch.load_attributes("./data/CAMELS_CH", basin_ids=["2004", "2007"])
|
|
66
|
+
|
|
67
|
+
# Load time series with filtering
|
|
68
|
+
data = camelsch.load_timeseries(
|
|
69
|
+
"./data/CAMELS_CH",
|
|
70
|
+
basin_ids=["2004"],
|
|
71
|
+
variables=["precipitation", "discharge_spec"],
|
|
72
|
+
start_date="1990-01-01",
|
|
73
|
+
end_date="2000-12-31",
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
# List available basins and variables
|
|
77
|
+
basins = camelsch.list_basins("./data/CAMELS_CH")
|
|
78
|
+
variables = camelsch.list_variables("./data/CAMELS_CH")
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## Data reference
|
|
82
|
+
|
|
83
|
+
CAMELS-CH provides daily hydrometeorological time series and static catchment attributes for 331 Swiss basins (1981-2020). Key variables include:
|
|
84
|
+
|
|
85
|
+
| Variable | Description |
|
|
86
|
+
|---|---|
|
|
87
|
+
| `precipitation` | Basin-mean daily precipitation (mm/d) |
|
|
88
|
+
| `temperature` | Basin-mean daily temperature (deg C) |
|
|
89
|
+
| `discharge_spec` | Observed specific discharge (mm/d) |
|
|
90
|
+
| `discharge_spec_sim` | Simulated specific discharge (mm/d) |
|
|
91
|
+
| `pet` / `pet_sim` | Potential evapotranspiration (mm/d) |
|
|
92
|
+
| `et` / `et_sim` | Actual evapotranspiration (mm/d) |
|
|
93
|
+
| `snow_water_eq_sim` | Simulated snow water equivalent (mm) |
|
|
94
|
+
|
|
95
|
+
Static attributes cover topography (`area`, `elev_mean`), climate (`p_mean`, `t_mean`), land cover, soil, and geology.
|
|
96
|
+
|
|
97
|
+
**Full dataset**: [CAMELS-CH on Zenodo](https://zenodo.org/records/15025258)
|
|
98
|
+
**Reference paper**: Hoege et al. (2023), *CAMELS-CH: hydro-meteorological time series and landscape attributes for 331 catchments in hydrologic Switzerland*, Earth Syst. Sci. Data.
|
|
99
|
+
|
|
100
|
+
## For developers
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
# Install dependencies
|
|
104
|
+
uv sync
|
|
105
|
+
|
|
106
|
+
# Run tests
|
|
107
|
+
uv run pytest
|
|
108
|
+
|
|
109
|
+
# Lint and format
|
|
110
|
+
uv run ruff check src/ tests/
|
|
111
|
+
uv run ruff format src/ tests/
|
|
112
|
+
|
|
113
|
+
# Type check
|
|
114
|
+
uv run mypy src/
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
## License
|
|
118
|
+
|
|
119
|
+
MIT
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "camelsch"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "CLI tool for extracting and working with CAMELS-CH hydrological data"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
authors = [
|
|
8
|
+
{ name = "Beatrice Marti", email = "mabesa@users.noreply.github.com" }
|
|
9
|
+
]
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"pandas>=2.0",
|
|
13
|
+
"typer>=0.9.0",
|
|
14
|
+
"rich>=13.0.0",
|
|
15
|
+
"pyarrow>=14.0.0",
|
|
16
|
+
]
|
|
17
|
+
|
|
18
|
+
[project.urls]
|
|
19
|
+
Homepage = "https://github.com/hydrosolutions/camelsch"
|
|
20
|
+
Repository = "https://github.com/hydrosolutions/camelsch"
|
|
21
|
+
Issues = "https://github.com/hydrosolutions/camelsch/issues"
|
|
22
|
+
|
|
23
|
+
[project.scripts]
|
|
24
|
+
camelsch = "camelsch.cli:app"
|
|
25
|
+
|
|
26
|
+
[build-system]
|
|
27
|
+
requires = ["uv_build>=0.9.13,<0.10.0"]
|
|
28
|
+
build-backend = "uv_build"
|
|
29
|
+
|
|
30
|
+
[tool.ruff]
|
|
31
|
+
target-version = "py310"
|
|
32
|
+
line-length = 99
|
|
33
|
+
|
|
34
|
+
[tool.ruff.lint]
|
|
35
|
+
select = ["E", "F", "W", "I", "N", "UP", "B", "A", "SIM", "RUF"]
|
|
36
|
+
ignore = ["B008"] # typer.Option() in defaults is the standard typer pattern
|
|
37
|
+
|
|
38
|
+
[tool.mypy]
|
|
39
|
+
python_version = "3.10"
|
|
40
|
+
strict = true
|
|
41
|
+
warn_return_any = true
|
|
42
|
+
warn_unused_configs = true
|
|
43
|
+
|
|
44
|
+
[[tool.mypy.overrides]]
|
|
45
|
+
module = ["pyarrow"]
|
|
46
|
+
ignore_missing_imports = true
|
|
47
|
+
|
|
48
|
+
[project.optional-dependencies]
|
|
49
|
+
dev = [
|
|
50
|
+
"mypy>=1.10",
|
|
51
|
+
"pandas-stubs>=2.0",
|
|
52
|
+
"pytest>=9.0.2",
|
|
53
|
+
"ruff>=0.15.2",
|
|
54
|
+
]
|
|
55
|
+
|
|
56
|
+
[tool.pytest.ini_options]
|
|
57
|
+
testpaths = ["tests", "src"]
|
|
58
|
+
addopts = "--doctest-modules"
|
|
59
|
+
|
|
60
|
+
[dependency-groups]
|
|
61
|
+
dev = [
|
|
62
|
+
"mypy>=1.10",
|
|
63
|
+
"pandas-stubs>=2.0",
|
|
64
|
+
"pytest>=9.0.2",
|
|
65
|
+
"ruff>=0.15.2",
|
|
66
|
+
]
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""camelsch — CLI tool for CAMELS-CH hydrological data extraction."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
|
|
5
|
+
from camelsch.attributes import get_attribute_names, load_attributes
|
|
6
|
+
from camelsch.download import download_camels_ch
|
|
7
|
+
from camelsch.export import export_attributes, export_merged, export_timeseries
|
|
8
|
+
from camelsch.timeseries import (
|
|
9
|
+
list_basins,
|
|
10
|
+
list_variables,
|
|
11
|
+
load_basin_timeseries,
|
|
12
|
+
load_timeseries,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
__version__ = "0.1.0"
|
|
16
|
+
|
|
17
|
+
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"download_camels_ch",
|
|
21
|
+
"export_attributes",
|
|
22
|
+
"export_merged",
|
|
23
|
+
"export_timeseries",
|
|
24
|
+
"get_attribute_names",
|
|
25
|
+
"list_basins",
|
|
26
|
+
"list_variables",
|
|
27
|
+
"load_attributes",
|
|
28
|
+
"load_basin_timeseries",
|
|
29
|
+
"load_timeseries",
|
|
30
|
+
]
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Load and merge static attribute CSVs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import pandas as pd
|
|
9
|
+
|
|
10
|
+
from camelsch.io import find_id_column, read_csv_robust, strip_units
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _read_attribute_csv(path: Path) -> pd.DataFrame:
|
|
16
|
+
"""Read a single CAMELS-CH attribute CSV handling encoding and comments."""
|
|
17
|
+
df = read_csv_robust(path)
|
|
18
|
+
df.columns = [strip_units(c) for c in df.columns]
|
|
19
|
+
|
|
20
|
+
id_col = find_id_column(df)
|
|
21
|
+
if id_col:
|
|
22
|
+
df[id_col] = df[id_col].astype(str)
|
|
23
|
+
if id_col != "gauge_id":
|
|
24
|
+
df = df.rename(columns={id_col: "gauge_id"})
|
|
25
|
+
df = df.set_index("gauge_id")
|
|
26
|
+
|
|
27
|
+
return df
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def load_attributes(
|
|
31
|
+
data_dir: str | Path,
|
|
32
|
+
basin_ids: list[str] | None = None,
|
|
33
|
+
) -> pd.DataFrame:
|
|
34
|
+
"""Load and merge all static attribute CSVs.
|
|
35
|
+
|
|
36
|
+
Reads every ``*.csv`` under ``static_attributes/`` (including subdirectories),
|
|
37
|
+
strips unit suffixes from column names, and merges on ``gauge_id``.
|
|
38
|
+
|
|
39
|
+
Args:
|
|
40
|
+
data_dir: Path to the extracted ``camels_ch`` directory.
|
|
41
|
+
basin_ids: Optional list of basin IDs to filter to.
|
|
42
|
+
|
|
43
|
+
Returns:
|
|
44
|
+
DataFrame indexed by ``gauge_id`` with all attribute columns.
|
|
45
|
+
"""
|
|
46
|
+
data_dir = Path(data_dir)
|
|
47
|
+
attr_dir = data_dir / "static_attributes"
|
|
48
|
+
|
|
49
|
+
csv_files = sorted(attr_dir.glob("*.csv"))
|
|
50
|
+
for subdir in ["supplements", "simulation_based"]:
|
|
51
|
+
sub = attr_dir / subdir
|
|
52
|
+
if sub.exists():
|
|
53
|
+
csv_files.extend(sorted(sub.glob("*.csv")))
|
|
54
|
+
|
|
55
|
+
if not csv_files:
|
|
56
|
+
msg = f"No attribute CSVs found in {attr_dir}"
|
|
57
|
+
raise FileNotFoundError(msg)
|
|
58
|
+
|
|
59
|
+
merged: pd.DataFrame | None = None
|
|
60
|
+
for csv_path in csv_files:
|
|
61
|
+
df = _read_attribute_csv(csv_path)
|
|
62
|
+
logger.debug("Loaded attribute file %s (%d cols)", csv_path.name, len(df.columns))
|
|
63
|
+
if merged is None:
|
|
64
|
+
merged = df
|
|
65
|
+
else:
|
|
66
|
+
new_cols = [c for c in df.columns if c not in merged.columns]
|
|
67
|
+
if new_cols:
|
|
68
|
+
merged = merged.join(df[new_cols], how="outer")
|
|
69
|
+
|
|
70
|
+
if merged is None:
|
|
71
|
+
msg = f"No attribute data could be loaded from {attr_dir}"
|
|
72
|
+
raise ValueError(msg)
|
|
73
|
+
|
|
74
|
+
if basin_ids is not None:
|
|
75
|
+
merged = merged.loc[merged.index.isin(basin_ids)]
|
|
76
|
+
|
|
77
|
+
logger.debug("Attributes loaded: %d basins, %d columns", len(merged), len(merged.columns))
|
|
78
|
+
return merged
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def get_attribute_names(data_dir: str | Path) -> list[str]:
|
|
82
|
+
"""List all available attribute column names."""
|
|
83
|
+
df = load_attributes(data_dir)
|
|
84
|
+
return list(df.columns)
|
|
@@ -0,0 +1,357 @@
|
|
|
1
|
+
"""CLI entry point for camelsch."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Literal
|
|
9
|
+
|
|
10
|
+
import pandas as pd
|
|
11
|
+
import typer
|
|
12
|
+
from rich.console import Console
|
|
13
|
+
from rich.table import Table
|
|
14
|
+
|
|
15
|
+
from camelsch import __version__
|
|
16
|
+
|
|
17
|
+
Format = Literal["csv", "parquet"]
|
|
18
|
+
|
|
19
|
+
app = typer.Typer(
|
|
20
|
+
name="camelsch",
|
|
21
|
+
help="CLI tool for CAMELS-CH hydrological data extraction.",
|
|
22
|
+
no_args_is_help=True,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
DEFAULT_DATA_DIR = Path("./data/CAMELS_CH")
|
|
26
|
+
|
|
27
|
+
console = Console()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _version_callback(value: bool) -> None:
|
|
31
|
+
if value:
|
|
32
|
+
typer.echo(f"camelsch {__version__}")
|
|
33
|
+
raise typer.Exit()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@app.callback()
|
|
37
|
+
def main(
|
|
38
|
+
version: bool = typer.Option(
|
|
39
|
+
False,
|
|
40
|
+
"--version",
|
|
41
|
+
"-V",
|
|
42
|
+
help="Show version and exit.",
|
|
43
|
+
callback=_version_callback,
|
|
44
|
+
is_eager=True,
|
|
45
|
+
),
|
|
46
|
+
) -> None:
|
|
47
|
+
"""CLI tool for CAMELS-CH hydrological data extraction."""
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def resolve_data_dir(data_dir: Path | None) -> Path:
|
|
51
|
+
"""Resolve data directory from argument, env var, or default."""
|
|
52
|
+
if data_dir is not None:
|
|
53
|
+
return data_dir
|
|
54
|
+
env = os.environ.get("CAMELSCH_DATA_DIR")
|
|
55
|
+
if env:
|
|
56
|
+
return Path(env)
|
|
57
|
+
return DEFAULT_DATA_DIR
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _check_data_dir(dd: Path) -> None:
|
|
61
|
+
"""Exit with a friendly hint if the data directory doesn't exist."""
|
|
62
|
+
if not dd.exists():
|
|
63
|
+
console.print(f"[red]Directory not found: {dd}[/red]")
|
|
64
|
+
console.print("Run 'camelsch download' first.")
|
|
65
|
+
raise typer.Exit(1)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _infer_format(output: Path, fmt: str) -> Format:
|
|
69
|
+
"""Infer output format from file extension, falling back to *fmt*."""
|
|
70
|
+
if output.suffix == ".parquet":
|
|
71
|
+
return "parquet"
|
|
72
|
+
if output.suffix == ".csv":
|
|
73
|
+
return "csv"
|
|
74
|
+
if fmt in ("csv", "parquet"):
|
|
75
|
+
return fmt # type: ignore[return-value]
|
|
76
|
+
return "csv"
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
# ---------------------------------------------------------------------------
|
|
80
|
+
# download
|
|
81
|
+
# ---------------------------------------------------------------------------
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@app.command()
|
|
85
|
+
def download(
|
|
86
|
+
dest: Path = typer.Option(DEFAULT_DATA_DIR, help="Target directory."),
|
|
87
|
+
force: bool = typer.Option(False, help="Re-download even if exists."),
|
|
88
|
+
url: str | None = typer.Option(None, help="Custom Zenodo URL to download from."),
|
|
89
|
+
) -> None:
|
|
90
|
+
"""Download and extract the CAMELS-CH dataset from Zenodo.
|
|
91
|
+
|
|
92
|
+
Example: camelsch download --dest ./my_data
|
|
93
|
+
"""
|
|
94
|
+
from camelsch.download import download_camels_ch
|
|
95
|
+
|
|
96
|
+
kwargs: dict[str, object] = {"dest": dest, "force": force}
|
|
97
|
+
if url is not None:
|
|
98
|
+
kwargs["url"] = url
|
|
99
|
+
path = download_camels_ch(**kwargs) # type: ignore[arg-type]
|
|
100
|
+
console.print(f"[green]Dataset ready at {path}[/green]")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
# ---------------------------------------------------------------------------
|
|
104
|
+
# info
|
|
105
|
+
# ---------------------------------------------------------------------------
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
@app.command()
|
|
109
|
+
def info(
|
|
110
|
+
data_dir: Path | None = typer.Option(None, help="Path to CAMELS_CH dir."),
|
|
111
|
+
) -> None:
|
|
112
|
+
"""Show dataset summary: basins, time range, variables.
|
|
113
|
+
|
|
114
|
+
Example: camelsch info --data-dir ./data/CAMELS_CH
|
|
115
|
+
"""
|
|
116
|
+
from camelsch.attributes import load_attributes
|
|
117
|
+
from camelsch.timeseries import (
|
|
118
|
+
list_basins,
|
|
119
|
+
list_variables,
|
|
120
|
+
load_basin_timeseries,
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
dd = resolve_data_dir(data_dir)
|
|
124
|
+
_check_data_dir(dd)
|
|
125
|
+
|
|
126
|
+
basin_ids = list_basins(dd)
|
|
127
|
+
variables = list_variables(dd)
|
|
128
|
+
|
|
129
|
+
date_range = ""
|
|
130
|
+
if basin_ids:
|
|
131
|
+
sample = load_basin_timeseries(dd, basin_ids[0])
|
|
132
|
+
if len(sample.index) > 0:
|
|
133
|
+
date_range = f"{sample.index.min().date()} to {sample.index.max().date()}"
|
|
134
|
+
|
|
135
|
+
try:
|
|
136
|
+
attrs = load_attributes(dd)
|
|
137
|
+
attr_count = len(attrs.columns)
|
|
138
|
+
except FileNotFoundError:
|
|
139
|
+
attr_count = 0
|
|
140
|
+
|
|
141
|
+
table = Table(title="CAMELS-CH Dataset Summary", show_header=False)
|
|
142
|
+
table.add_column("Key", style="bold")
|
|
143
|
+
table.add_column("Value")
|
|
144
|
+
table.add_row("Location", str(dd))
|
|
145
|
+
table.add_row("Basins", str(len(basin_ids)))
|
|
146
|
+
table.add_row("Time range", date_range)
|
|
147
|
+
table.add_row("Variables", ", ".join(variables))
|
|
148
|
+
table.add_row("Attributes", f"{attr_count} static features")
|
|
149
|
+
console.print(table)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
# ---------------------------------------------------------------------------
|
|
153
|
+
# basins
|
|
154
|
+
# ---------------------------------------------------------------------------
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
@app.command()
|
|
158
|
+
def basins(
|
|
159
|
+
data_dir: Path | None = typer.Option(None, help="Path to CAMELS_CH dir."),
|
|
160
|
+
fmt: str = typer.Option("table", "--format", help="Output: table, csv, json."),
|
|
161
|
+
attrs: str | None = typer.Option(None, help="Comma-separated attribute names."),
|
|
162
|
+
) -> None:
|
|
163
|
+
"""List available basin IDs with optional attributes.
|
|
164
|
+
|
|
165
|
+
Example: camelsch basins --format json --attrs area,p_mean
|
|
166
|
+
"""
|
|
167
|
+
from camelsch.timeseries import list_basins as _list_basins
|
|
168
|
+
|
|
169
|
+
dd = resolve_data_dir(data_dir)
|
|
170
|
+
_check_data_dir(dd)
|
|
171
|
+
basin_ids = _list_basins(dd)
|
|
172
|
+
|
|
173
|
+
attr_df = None
|
|
174
|
+
if attrs:
|
|
175
|
+
from camelsch.attributes import load_attributes
|
|
176
|
+
|
|
177
|
+
attr_names = [a.strip() for a in attrs.split(",")]
|
|
178
|
+
attr_df = load_attributes(dd)
|
|
179
|
+
valid_cols = [c for c in attr_names if c in attr_df.columns]
|
|
180
|
+
attr_df = attr_df[valid_cols] if valid_cols else attr_df.iloc[:, :0]
|
|
181
|
+
attr_df = attr_df.loc[attr_df.index.isin(basin_ids)]
|
|
182
|
+
|
|
183
|
+
if fmt == "json":
|
|
184
|
+
_basins_json(basin_ids, attr_df)
|
|
185
|
+
elif fmt == "csv":
|
|
186
|
+
_basins_csv(basin_ids, attr_df)
|
|
187
|
+
else:
|
|
188
|
+
_basins_table(basin_ids, attr_df)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _basins_json(basin_ids: list[str], attr_df: pd.DataFrame | None) -> None:
|
|
192
|
+
"""Output basin list as JSON."""
|
|
193
|
+
if attr_df is not None:
|
|
194
|
+
records = []
|
|
195
|
+
for bid in basin_ids:
|
|
196
|
+
rec: dict[str, object] = {"basin_id": bid}
|
|
197
|
+
if bid in attr_df.index:
|
|
198
|
+
rec.update(attr_df.loc[bid].to_dict()) # type: ignore[arg-type]
|
|
199
|
+
records.append(rec)
|
|
200
|
+
typer.echo(json.dumps(records, indent=2))
|
|
201
|
+
else:
|
|
202
|
+
typer.echo(json.dumps(basin_ids, indent=2))
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _basins_csv(basin_ids: list[str], attr_df: pd.DataFrame | None) -> None:
|
|
206
|
+
"""Output basin list as CSV."""
|
|
207
|
+
if attr_df is not None:
|
|
208
|
+
header = "basin_id," + ",".join(attr_df.columns)
|
|
209
|
+
typer.echo(header)
|
|
210
|
+
for bid in basin_ids:
|
|
211
|
+
if bid in attr_df.index:
|
|
212
|
+
vals = ",".join(str(v) for v in attr_df.loc[bid])
|
|
213
|
+
typer.echo(f"{bid},{vals}")
|
|
214
|
+
else:
|
|
215
|
+
typer.echo(bid)
|
|
216
|
+
else:
|
|
217
|
+
for bid in basin_ids:
|
|
218
|
+
typer.echo(bid)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _basins_table(basin_ids: list[str], attr_df: pd.DataFrame | None) -> None:
|
|
222
|
+
"""Output basin list as a rich table."""
|
|
223
|
+
table = Table(title="CAMELS-CH Basins")
|
|
224
|
+
table.add_column("basin_id")
|
|
225
|
+
if attr_df is not None:
|
|
226
|
+
for col in attr_df.columns:
|
|
227
|
+
table.add_column(col)
|
|
228
|
+
for bid in basin_ids:
|
|
229
|
+
if bid in attr_df.index:
|
|
230
|
+
row = attr_df.loc[bid]
|
|
231
|
+
table.add_row(bid, *(str(v) for v in row))
|
|
232
|
+
else:
|
|
233
|
+
table.add_row(bid, *[""] * len(attr_df.columns))
|
|
234
|
+
else:
|
|
235
|
+
for bid in basin_ids:
|
|
236
|
+
table.add_row(bid)
|
|
237
|
+
console.print(table)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
# ---------------------------------------------------------------------------
|
|
241
|
+
# attributes
|
|
242
|
+
# ---------------------------------------------------------------------------
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
@app.command()
|
|
246
|
+
def attributes(
|
|
247
|
+
data_dir: Path | None = typer.Option(None, help="Path to CAMELS_CH dir."),
|
|
248
|
+
basins_opt: str | None = typer.Option(None, "--basins", help="Comma-separated basin IDs."),
|
|
249
|
+
output: Path | None = typer.Option(None, help="Output file path."),
|
|
250
|
+
fmt: str = typer.Option("csv", "--format", help="Output format: csv or parquet."),
|
|
251
|
+
) -> None:
|
|
252
|
+
"""Extract static catchment attributes to CSV or Parquet.
|
|
253
|
+
|
|
254
|
+
Example: camelsch attributes --basins 2004,2007 --output attrs.csv
|
|
255
|
+
"""
|
|
256
|
+
from camelsch.attributes import load_attributes
|
|
257
|
+
from camelsch.export import export_attributes
|
|
258
|
+
|
|
259
|
+
dd = resolve_data_dir(data_dir)
|
|
260
|
+
_check_data_dir(dd)
|
|
261
|
+
bid_list = [b.strip() for b in basins_opt.split(",")] if basins_opt else None
|
|
262
|
+
df = load_attributes(dd, basin_ids=bid_list)
|
|
263
|
+
|
|
264
|
+
if output:
|
|
265
|
+
export_attributes(df, output, fmt=_infer_format(output, fmt))
|
|
266
|
+
console.print(f"[green]Attributes written to {output}[/green]")
|
|
267
|
+
else:
|
|
268
|
+
typer.echo(df.to_csv())
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
# ---------------------------------------------------------------------------
|
|
272
|
+
# timeseries
|
|
273
|
+
# ---------------------------------------------------------------------------
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
@app.command()
|
|
277
|
+
def timeseries(
|
|
278
|
+
data_dir: Path | None = typer.Option(None, help="Path to CAMELS_CH dir."),
|
|
279
|
+
basins_opt: str | None = typer.Option(
|
|
280
|
+
None, "--basins", help="Comma-separated basin IDs, or 'all'."
|
|
281
|
+
),
|
|
282
|
+
vars_opt: str | None = typer.Option(None, "--vars", help="Comma-separated variable names."),
|
|
283
|
+
start: str | None = typer.Option(None, help="Start date (YYYY-MM-DD)."),
|
|
284
|
+
end: str | None = typer.Option(None, help="End date (YYYY-MM-DD)."),
|
|
285
|
+
output: Path | None = typer.Option(None, help="Output file path."),
|
|
286
|
+
fmt: str = typer.Option("csv", "--format", help="Output format: csv or parquet."),
|
|
287
|
+
) -> None:
|
|
288
|
+
"""Extract time series data for one or more basins.
|
|
289
|
+
|
|
290
|
+
Example: camelsch timeseries --basins 2004 --vars precipitation --start 1990-01-01
|
|
291
|
+
"""
|
|
292
|
+
from camelsch.export import export_timeseries
|
|
293
|
+
from camelsch.timeseries import load_timeseries
|
|
294
|
+
|
|
295
|
+
dd = resolve_data_dir(data_dir)
|
|
296
|
+
_check_data_dir(dd)
|
|
297
|
+
bid_list = (
|
|
298
|
+
None
|
|
299
|
+
if basins_opt is None or basins_opt.strip().lower() == "all"
|
|
300
|
+
else [b.strip() for b in basins_opt.split(",")]
|
|
301
|
+
)
|
|
302
|
+
var_list = [v.strip() for v in vars_opt.split(",")] if vars_opt else None
|
|
303
|
+
|
|
304
|
+
data = load_timeseries(
|
|
305
|
+
dd,
|
|
306
|
+
basin_ids=bid_list,
|
|
307
|
+
variables=var_list,
|
|
308
|
+
start_date=start,
|
|
309
|
+
end_date=end,
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
if output:
|
|
313
|
+
export_timeseries(data, output, fmt=_infer_format(output, fmt))
|
|
314
|
+
console.print(f"[green]Time series written to {output}[/green]")
|
|
315
|
+
else:
|
|
316
|
+
for bid, df in data.items():
|
|
317
|
+
if len(data) > 1:
|
|
318
|
+
typer.echo(f"--- Basin {bid} ---")
|
|
319
|
+
typer.echo(df.to_csv())
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
# ---------------------------------------------------------------------------
|
|
323
|
+
# export
|
|
324
|
+
# ---------------------------------------------------------------------------
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
@app.command()
|
|
328
|
+
def export(
|
|
329
|
+
data_dir: Path | None = typer.Option(None, help="Path to CAMELS_CH dir."),
|
|
330
|
+
basins_opt: str | None = typer.Option(None, "--basins", help="Comma-separated basin IDs."),
|
|
331
|
+
output: Path = typer.Option(Path("camels_ch_export.parquet"), help="Output file path."),
|
|
332
|
+
fmt: str = typer.Option("parquet", "--format", help="Output format: csv or parquet."),
|
|
333
|
+
include_attrs: bool = typer.Option(False, help="Merge static attributes into output."),
|
|
334
|
+
) -> None:
|
|
335
|
+
"""Batch export: merge time series (+ optional attributes) into one file.
|
|
336
|
+
|
|
337
|
+
Example: camelsch export --basins 2004,2007 --include-attrs --output merged.parquet
|
|
338
|
+
"""
|
|
339
|
+
from camelsch.attributes import load_attributes
|
|
340
|
+
from camelsch.export import export_merged, export_timeseries
|
|
341
|
+
from camelsch.timeseries import load_timeseries
|
|
342
|
+
|
|
343
|
+
dd = resolve_data_dir(data_dir)
|
|
344
|
+
_check_data_dir(dd)
|
|
345
|
+
bid_list = [b.strip() for b in basins_opt.split(",")] if basins_opt else None
|
|
346
|
+
|
|
347
|
+
data = load_timeseries(dd, basin_ids=bid_list)
|
|
348
|
+
|
|
349
|
+
resolved_fmt = _infer_format(output, fmt)
|
|
350
|
+
|
|
351
|
+
if include_attrs:
|
|
352
|
+
attrs = load_attributes(dd, basin_ids=bid_list)
|
|
353
|
+
export_merged(data, attrs, output, fmt=resolved_fmt)
|
|
354
|
+
else:
|
|
355
|
+
export_timeseries(data, output, fmt=resolved_fmt)
|
|
356
|
+
|
|
357
|
+
console.print(f"[green]Exported {len(data)} basin(s) to {output}[/green]")
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""Download and extract the CAMELS-CH dataset."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import shutil
|
|
7
|
+
import zipfile
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from urllib.request import urlopen
|
|
10
|
+
|
|
11
|
+
from rich.progress import (
|
|
12
|
+
BarColumn,
|
|
13
|
+
DownloadColumn,
|
|
14
|
+
Progress,
|
|
15
|
+
TextColumn,
|
|
16
|
+
TransferSpeedColumn,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
logger = logging.getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
CAMELS_CH_URL = "https://zenodo.org/api/records/15025258/files/camels_ch.zip/content"
|
|
22
|
+
ZIP_FILENAME = "camels_ch.zip"
|
|
23
|
+
EXTRACTED_DIR = "CAMELS_CH"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def download_camels_ch(
|
|
27
|
+
dest: Path | str = Path("./data/CAMELS_CH"),
|
|
28
|
+
url: str = CAMELS_CH_URL,
|
|
29
|
+
force: bool = False,
|
|
30
|
+
) -> Path:
|
|
31
|
+
"""Download and extract CAMELS-CH. Returns path to extracted dir.
|
|
32
|
+
|
|
33
|
+
Args:
|
|
34
|
+
dest: Target directory for the extracted dataset.
|
|
35
|
+
url: URL to download from.
|
|
36
|
+
force: Re-download even if already exists.
|
|
37
|
+
|
|
38
|
+
Returns:
|
|
39
|
+
Path to the extracted dataset directory.
|
|
40
|
+
"""
|
|
41
|
+
dest = Path(dest)
|
|
42
|
+
|
|
43
|
+
if dest.exists() and not force:
|
|
44
|
+
logger.debug("Dataset already exists at %s, skipping download", dest)
|
|
45
|
+
return dest
|
|
46
|
+
|
|
47
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
48
|
+
zip_path = dest.parent / ZIP_FILENAME
|
|
49
|
+
|
|
50
|
+
# Download with rich progress bar
|
|
51
|
+
if not zip_path.exists() or force:
|
|
52
|
+
logger.debug("Downloading CAMELS-CH from %s", url)
|
|
53
|
+
_download_with_progress(url, zip_path)
|
|
54
|
+
|
|
55
|
+
# Extract
|
|
56
|
+
_extract_and_rename(zip_path, dest)
|
|
57
|
+
|
|
58
|
+
# Clean up zip
|
|
59
|
+
zip_path.unlink(missing_ok=True)
|
|
60
|
+
|
|
61
|
+
return dest
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _validate_zip_members(zf: zipfile.ZipFile, target: Path) -> None:
|
|
65
|
+
"""Raise ValueError if any zip member would extract outside *target*."""
|
|
66
|
+
resolved = target.resolve()
|
|
67
|
+
for member in zf.namelist():
|
|
68
|
+
member_path = (target / member).resolve()
|
|
69
|
+
if not str(member_path).startswith(str(resolved) + "/") and member_path != resolved:
|
|
70
|
+
msg = f"Zip member {member!r} would escape target directory"
|
|
71
|
+
raise ValueError(msg)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _download_with_progress(url: str, zip_path: Path) -> None:
|
|
75
|
+
"""Download a file with a rich progress bar."""
|
|
76
|
+
response = urlopen(url, timeout=30)
|
|
77
|
+
total = int(response.headers.get("Content-Length", 0))
|
|
78
|
+
|
|
79
|
+
with Progress(
|
|
80
|
+
TextColumn("[bold blue]{task.description}"),
|
|
81
|
+
BarColumn(),
|
|
82
|
+
DownloadColumn(),
|
|
83
|
+
TransferSpeedColumn(),
|
|
84
|
+
) as progress:
|
|
85
|
+
task = progress.add_task("Downloading CAMELS-CH", total=total or None)
|
|
86
|
+
with open(zip_path, "wb") as f:
|
|
87
|
+
while chunk := response.read(8192):
|
|
88
|
+
f.write(chunk)
|
|
89
|
+
progress.update(task, advance=len(chunk))
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _extract_and_rename(zip_path: Path, dest: Path) -> None:
|
|
93
|
+
"""Extract zip and rename extracted folder to dest."""
|
|
94
|
+
extract_to = dest.parent
|
|
95
|
+
with zipfile.ZipFile(zip_path, "r") as zf:
|
|
96
|
+
_validate_zip_members(zf, extract_to)
|
|
97
|
+
zf.extractall(extract_to)
|
|
98
|
+
|
|
99
|
+
# Auto-detect extracted folder (starts with "camels" or "CAMELS")
|
|
100
|
+
for item in extract_to.iterdir():
|
|
101
|
+
if item.is_dir() and item.name.lower().startswith("camels") and item != dest:
|
|
102
|
+
if dest.exists():
|
|
103
|
+
shutil.rmtree(dest)
|
|
104
|
+
item.rename(dest)
|
|
105
|
+
return
|
|
106
|
+
|
|
107
|
+
# If extraction produced files directly (no subfolder), dest should already exist
|
|
108
|
+
if not dest.exists():
|
|
109
|
+
msg = f"Could not find extracted CAMELS-CH folder in {extract_to}"
|
|
110
|
+
raise FileNotFoundError(msg)
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""Export time series and attributes to CSV/Parquet."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Literal
|
|
8
|
+
|
|
9
|
+
import pandas as pd
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
Format = Literal["csv", "parquet"]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def export_timeseries(
|
|
17
|
+
data: dict[str, pd.DataFrame],
|
|
18
|
+
output: Path,
|
|
19
|
+
fmt: Format = "csv",
|
|
20
|
+
) -> None:
|
|
21
|
+
"""Export time series dict to file(s).
|
|
22
|
+
|
|
23
|
+
For parquet: single file with ``basin_id`` column.
|
|
24
|
+
For CSV: single file if one basin, directory of files if multiple.
|
|
25
|
+
"""
|
|
26
|
+
logger.debug("Exporting %d basin(s) as %s to %s", len(data), fmt, output)
|
|
27
|
+
if fmt == "parquet":
|
|
28
|
+
frames = []
|
|
29
|
+
for bid, df in data.items():
|
|
30
|
+
df = df.copy()
|
|
31
|
+
df.insert(0, "basin_id", bid)
|
|
32
|
+
frames.append(df)
|
|
33
|
+
combined = pd.concat(frames)
|
|
34
|
+
combined.to_parquet(output)
|
|
35
|
+
else:
|
|
36
|
+
if len(data) == 1:
|
|
37
|
+
bid, df = next(iter(data.items()))
|
|
38
|
+
df.to_csv(output)
|
|
39
|
+
else:
|
|
40
|
+
output.mkdir(parents=True, exist_ok=True)
|
|
41
|
+
for bid, df in data.items():
|
|
42
|
+
df.to_csv(output / f"basin_{bid}.csv")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def export_attributes(
|
|
46
|
+
df: pd.DataFrame,
|
|
47
|
+
output: Path,
|
|
48
|
+
fmt: Format = "csv",
|
|
49
|
+
) -> None:
|
|
50
|
+
"""Export attributes DataFrame to file."""
|
|
51
|
+
if fmt == "parquet":
|
|
52
|
+
df.to_parquet(output)
|
|
53
|
+
else:
|
|
54
|
+
df.to_csv(output)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def export_merged(
|
|
58
|
+
timeseries: dict[str, pd.DataFrame],
|
|
59
|
+
attributes: pd.DataFrame,
|
|
60
|
+
output: Path,
|
|
61
|
+
fmt: Format = "parquet",
|
|
62
|
+
) -> None:
|
|
63
|
+
"""Export merged timeseries + attributes as flat analysis-ready file.
|
|
64
|
+
|
|
65
|
+
Produces a dataset with columns: basin_id, date, <forcing_vars>, <target>, [<static_attrs>].
|
|
66
|
+
Static attributes are repeated for each row of a given basin.
|
|
67
|
+
"""
|
|
68
|
+
logger.debug("Exporting merged data for %d basin(s) as %s to %s", len(timeseries), fmt, output)
|
|
69
|
+
frames = []
|
|
70
|
+
for bid, df in timeseries.items():
|
|
71
|
+
df = df.copy()
|
|
72
|
+
df.insert(0, "basin_id", bid)
|
|
73
|
+
|
|
74
|
+
if bid in attributes.index:
|
|
75
|
+
for col in attributes.columns:
|
|
76
|
+
df[col] = attributes.loc[bid, col]
|
|
77
|
+
|
|
78
|
+
frames.append(df)
|
|
79
|
+
|
|
80
|
+
combined = pd.concat(frames)
|
|
81
|
+
|
|
82
|
+
if fmt == "parquet":
|
|
83
|
+
combined.to_parquet(output)
|
|
84
|
+
else:
|
|
85
|
+
combined.to_csv(output)
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""Shared I/O helpers for reading CAMELS-CH CSV files."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
import pandas as pd
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def strip_units(name: str) -> str:
|
|
13
|
+
"""Strip parenthesised unit suffixes from a column name.
|
|
14
|
+
|
|
15
|
+
Examples:
|
|
16
|
+
>>> strip_units("discharge_spec(mm/d)")
|
|
17
|
+
'discharge_spec'
|
|
18
|
+
>>> strip_units("temperature_mean(degC)")
|
|
19
|
+
'temperature_mean'
|
|
20
|
+
>>> strip_units("gauge_id")
|
|
21
|
+
'gauge_id'
|
|
22
|
+
"""
|
|
23
|
+
return re.sub(r"\(.*?\)\s*$", "", name).strip()
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def read_csv_robust(filepath: Path, **kwargs: Any) -> pd.DataFrame:
|
|
27
|
+
"""Read CSV trying utf-8, then latin-1 encoding. Handle comment lines."""
|
|
28
|
+
for encoding in ("utf-8", "latin-1"):
|
|
29
|
+
try:
|
|
30
|
+
df: pd.DataFrame = pd.read_csv(filepath, comment="#", encoding=encoding, **kwargs)
|
|
31
|
+
return df
|
|
32
|
+
except UnicodeDecodeError:
|
|
33
|
+
continue
|
|
34
|
+
msg = f"Could not decode {filepath} with utf-8 or latin-1"
|
|
35
|
+
raise ValueError(msg)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def find_id_column(df: pd.DataFrame) -> str | None:
|
|
39
|
+
"""Detect basin/gauge ID column from common names.
|
|
40
|
+
|
|
41
|
+
Examples:
|
|
42
|
+
>>> import pandas as pd
|
|
43
|
+
>>> find_id_column(pd.DataFrame({"gauge_id": [1], "value": [2]}))
|
|
44
|
+
'gauge_id'
|
|
45
|
+
>>> find_id_column(pd.DataFrame({"basin_id": [1], "value": [2]}))
|
|
46
|
+
'basin_id'
|
|
47
|
+
>>> find_id_column(pd.DataFrame({"x": [1], "y": [2]})) is None
|
|
48
|
+
True
|
|
49
|
+
"""
|
|
50
|
+
candidates = ["gauge_id", "basin_id", "id", "ID", "station_id", "catchment_id"]
|
|
51
|
+
for name in candidates:
|
|
52
|
+
if name in df.columns:
|
|
53
|
+
return name
|
|
54
|
+
return None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def find_date_column(df: pd.DataFrame) -> str | None:
|
|
58
|
+
"""Detect date/time column from common names.
|
|
59
|
+
|
|
60
|
+
Examples:
|
|
61
|
+
>>> import pandas as pd
|
|
62
|
+
>>> find_date_column(pd.DataFrame({"date": ["2020-01-01"], "value": [1]}))
|
|
63
|
+
'date'
|
|
64
|
+
>>> find_date_column(pd.DataFrame({"timestamp": ["2020-01-01"]}))
|
|
65
|
+
'timestamp'
|
|
66
|
+
>>> find_date_column(pd.DataFrame({"x": [1]})) is None
|
|
67
|
+
True
|
|
68
|
+
"""
|
|
69
|
+
candidates = ["date", "Date", "datetime", "time", "timestamp"]
|
|
70
|
+
for name in candidates:
|
|
71
|
+
if name in df.columns:
|
|
72
|
+
return name
|
|
73
|
+
return None
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def extract_basin_id(filepath: Path) -> str:
|
|
77
|
+
"""Extract numeric basin ID from filename.
|
|
78
|
+
|
|
79
|
+
Expects filenames like ``CAMELS_CH_obs_based_2034.csv`` → ``"2034"``.
|
|
80
|
+
|
|
81
|
+
Examples:
|
|
82
|
+
>>> extract_basin_id(Path("CAMELS_CH_obs_based_2034.csv"))
|
|
83
|
+
'2034'
|
|
84
|
+
>>> extract_basin_id(Path("/data/timeseries/sim_2007.csv"))
|
|
85
|
+
'2007'
|
|
86
|
+
"""
|
|
87
|
+
return filepath.stem.rsplit("_", 1)[-1]
|
|
File without changes
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"""Load and merge observation + simulation time series."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import pandas as pd
|
|
9
|
+
|
|
10
|
+
from camelsch.io import extract_basin_id, find_date_column, read_csv_robust, strip_units
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _read_ts_csv(path: Path) -> pd.DataFrame:
|
|
16
|
+
"""Read a single time series CSV with encoding fallback and unit stripping."""
|
|
17
|
+
df = read_csv_robust(path)
|
|
18
|
+
df.columns = [strip_units(c) for c in df.columns]
|
|
19
|
+
|
|
20
|
+
date_col = find_date_column(df)
|
|
21
|
+
if date_col:
|
|
22
|
+
df[date_col] = pd.to_datetime(df[date_col])
|
|
23
|
+
if date_col != "date":
|
|
24
|
+
df = df.rename(columns={date_col: "date"})
|
|
25
|
+
df = df.set_index("date")
|
|
26
|
+
|
|
27
|
+
# Drop gauge_id column from data — it's redundant in per-basin files
|
|
28
|
+
if "gauge_id" in df.columns:
|
|
29
|
+
df = df.drop(columns=["gauge_id"])
|
|
30
|
+
|
|
31
|
+
return df
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _add_aliases(df: pd.DataFrame) -> pd.DataFrame:
|
|
35
|
+
"""Add convenience aliases: pet_sim -> pet, et_sim -> et."""
|
|
36
|
+
if "pet_sim" in df.columns and "pet" not in df.columns:
|
|
37
|
+
df["pet"] = df["pet_sim"]
|
|
38
|
+
if "et_sim" in df.columns and "et" not in df.columns:
|
|
39
|
+
df["et"] = df["et_sim"]
|
|
40
|
+
return df
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _find_timeseries_dirs(data_dir: Path) -> tuple[Path | None, Path | None]:
|
|
44
|
+
"""Find observation_based/ and simulation_based/ directories."""
|
|
45
|
+
ts_dir = data_dir / "timeseries"
|
|
46
|
+
obs_dir = ts_dir / "observation_based"
|
|
47
|
+
sim_dir = ts_dir / "simulation_based"
|
|
48
|
+
return (
|
|
49
|
+
obs_dir if obs_dir.exists() else None,
|
|
50
|
+
sim_dir if sim_dir.exists() else None,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def list_basins(data_dir: str | Path) -> list[str]:
|
|
55
|
+
"""List all available basin IDs from time series filenames."""
|
|
56
|
+
data_dir = Path(data_dir)
|
|
57
|
+
obs_dir, sim_dir = _find_timeseries_dirs(data_dir)
|
|
58
|
+
|
|
59
|
+
ids: set[str] = set()
|
|
60
|
+
for d in (obs_dir, sim_dir):
|
|
61
|
+
if d is not None:
|
|
62
|
+
for f in d.glob("*.csv"):
|
|
63
|
+
ids.add(extract_basin_id(f))
|
|
64
|
+
return sorted(ids)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def list_variables(data_dir: str | Path) -> list[str]:
|
|
68
|
+
"""List all available variable names from a sample basin."""
|
|
69
|
+
data_dir = Path(data_dir)
|
|
70
|
+
basin_ids = list_basins(data_dir)
|
|
71
|
+
if not basin_ids:
|
|
72
|
+
return []
|
|
73
|
+
sample = load_basin_timeseries(data_dir, basin_ids[0])
|
|
74
|
+
# Exclude aliases from the canonical list
|
|
75
|
+
return [c for c in sample.columns if c not in ("pet", "et")]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def load_basin_timeseries(data_dir: str | Path, basin_id: str) -> pd.DataFrame:
|
|
79
|
+
"""Load and merge obs + sim time series for one basin.
|
|
80
|
+
|
|
81
|
+
Args:
|
|
82
|
+
data_dir: Path to the extracted ``camels_ch`` directory.
|
|
83
|
+
basin_id: 4-digit FOEN gauge identifier (e.g. ``"2004"``).
|
|
84
|
+
|
|
85
|
+
Returns:
|
|
86
|
+
DataFrame with DatetimeIndex and all observation + simulation columns.
|
|
87
|
+
"""
|
|
88
|
+
data_dir = Path(data_dir)
|
|
89
|
+
obs_dir, sim_dir = _find_timeseries_dirs(data_dir)
|
|
90
|
+
|
|
91
|
+
obs: pd.DataFrame | None = None
|
|
92
|
+
sim: pd.DataFrame | None = None
|
|
93
|
+
|
|
94
|
+
if obs_dir:
|
|
95
|
+
matches = list(obs_dir.glob(f"*_{basin_id}.csv"))
|
|
96
|
+
if matches:
|
|
97
|
+
obs = _read_ts_csv(matches[0])
|
|
98
|
+
|
|
99
|
+
if sim_dir:
|
|
100
|
+
matches = list(sim_dir.glob(f"*_{basin_id}.csv"))
|
|
101
|
+
if matches:
|
|
102
|
+
sim = _read_ts_csv(matches[0])
|
|
103
|
+
|
|
104
|
+
if obs is None and sim is None:
|
|
105
|
+
msg = f"No time series files found for basin {basin_id}"
|
|
106
|
+
raise FileNotFoundError(msg)
|
|
107
|
+
|
|
108
|
+
if obs is not None and sim is not None:
|
|
109
|
+
merged = obs.join(sim, how="outer")
|
|
110
|
+
logger.debug("Merged obs+sim for basin %s (%d rows)", basin_id, len(merged))
|
|
111
|
+
elif obs is not None:
|
|
112
|
+
merged = obs
|
|
113
|
+
else:
|
|
114
|
+
merged = sim # both-None handled above
|
|
115
|
+
|
|
116
|
+
return _add_aliases(merged)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def load_timeseries(
|
|
120
|
+
data_dir: str | Path,
|
|
121
|
+
basin_ids: list[str] | None = None,
|
|
122
|
+
variables: list[str] | None = None,
|
|
123
|
+
start_date: str | None = None,
|
|
124
|
+
end_date: str | None = None,
|
|
125
|
+
) -> dict[str, pd.DataFrame]:
|
|
126
|
+
"""Load time series for multiple basins with optional filtering.
|
|
127
|
+
|
|
128
|
+
Args:
|
|
129
|
+
data_dir: Path to the extracted ``camels_ch`` directory.
|
|
130
|
+
basin_ids: List of basin IDs to load. If ``None``, loads all.
|
|
131
|
+
variables: Variables to include. If ``None``, includes all.
|
|
132
|
+
start_date: Start date filter (YYYY-MM-DD).
|
|
133
|
+
end_date: End date filter (YYYY-MM-DD).
|
|
134
|
+
|
|
135
|
+
Returns:
|
|
136
|
+
Dict mapping basin ID to its merged time series DataFrame.
|
|
137
|
+
"""
|
|
138
|
+
data_dir = Path(data_dir)
|
|
139
|
+
|
|
140
|
+
if basin_ids is None:
|
|
141
|
+
basin_ids = list_basins(data_dir)
|
|
142
|
+
if not basin_ids:
|
|
143
|
+
msg = f"No time series files found in {data_dir / 'timeseries'}"
|
|
144
|
+
raise FileNotFoundError(msg)
|
|
145
|
+
|
|
146
|
+
result: dict[str, pd.DataFrame] = {}
|
|
147
|
+
for bid in basin_ids:
|
|
148
|
+
df = load_basin_timeseries(data_dir, bid)
|
|
149
|
+
|
|
150
|
+
if start_date:
|
|
151
|
+
df = df.loc[start_date:] # type: ignore[misc]
|
|
152
|
+
if end_date:
|
|
153
|
+
df = df.loc[:end_date] # type: ignore[misc]
|
|
154
|
+
if variables:
|
|
155
|
+
# Also include aliases if the base variable is requested
|
|
156
|
+
cols = []
|
|
157
|
+
for v in variables:
|
|
158
|
+
if v in df.columns:
|
|
159
|
+
cols.append(v)
|
|
160
|
+
df = df[cols] if cols else df
|
|
161
|
+
|
|
162
|
+
result[bid] = df
|
|
163
|
+
|
|
164
|
+
return result
|