dwdopen 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dwdopen-0.2.1/.claude/hooks/post-tool-call.py +102 -0
- dwdopen-0.2.1/.claude/settings.local.json +69 -0
- dwdopen-0.2.1/.gitignore +12 -0
- dwdopen-0.2.1/CHANGELOG.md +56 -0
- dwdopen-0.2.1/LICENSE +21 -0
- dwdopen-0.2.1/PKG-INFO +226 -0
- dwdopen-0.2.1/README.md +199 -0
- dwdopen-0.2.1/design/internal_api_mock.py +220 -0
- dwdopen-0.2.1/design/usage_examples.py +172 -0
- dwdopen-0.2.1/pyproject.toml +56 -0
- dwdopen-0.2.1/src/dwdopen/__init__.py +61 -0
- dwdopen-0.2.1/src/dwdopen/_fileserver/__init__.py +4 -0
- dwdopen-0.2.1/src/dwdopen/_fileserver/download.py +265 -0
- dwdopen-0.2.1/src/dwdopen/_fileserver/http.py +60 -0
- dwdopen-0.2.1/src/dwdopen/_fileserver/listing.py +108 -0
- dwdopen-0.2.1/src/dwdopen/_fileserver/naming.py +70 -0
- dwdopen-0.2.1/src/dwdopen/_fileserver/paths.py +48 -0
- dwdopen-0.2.1/src/dwdopen/_fileserver/traversal.py +404 -0
- dwdopen-0.2.1/src/dwdopen/client.py +164 -0
- dwdopen-0.2.1/src/dwdopen/exceptions.py +147 -0
- dwdopen-0.2.1/src/dwdopen/nwp/__init__.py +22 -0
- dwdopen-0.2.1/src/dwdopen/nwp/catalogue.py +125 -0
- dwdopen-0.2.1/src/dwdopen/nwp/durations.py +165 -0
- dwdopen-0.2.1/src/dwdopen/nwp/model.py +244 -0
- dwdopen-0.2.1/src/dwdopen/nwp/query.py +395 -0
- dwdopen-0.2.1/src/dwdopen/nwp/request.py +492 -0
- dwdopen-0.2.1/src/dwdopen/nwp/run.py +68 -0
- dwdopen-0.2.1/src/dwdopen/nwp/selectors.py +176 -0
- dwdopen-0.2.1/src/dwdopen/py.typed +0 -0
- dwdopen-0.2.1/testing.py +53 -0
- dwdopen-0.2.1/tests/integration/test_live_discovery.py +18 -0
- dwdopen-0.2.1/tests/integration/test_live_download.py +175 -0
- dwdopen-0.2.1/tests/integration/test_live_listing.py +16 -0
- dwdopen-0.2.1/tests/unit/test_case_insensitivity.py +84 -0
- dwdopen-0.2.1/tests/unit/test_download.py +379 -0
- dwdopen-0.2.1/tests/unit/test_durations.py +211 -0
- dwdopen-0.2.1/tests/unit/test_levels.py +218 -0
- dwdopen-0.2.1/tests/unit/test_listing.py +55 -0
- dwdopen-0.2.1/tests/unit/test_members.py +160 -0
- dwdopen-0.2.1/tests/unit/test_naming_plans.py +205 -0
- dwdopen-0.2.1/tests/unit/test_paths.py +63 -0
- dwdopen-0.2.1/tests/unit/test_repr.py +130 -0
- dwdopen-0.2.1/tests/unit/test_traversal.py +181 -0
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import hashlib
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import sys
|
|
5
|
+
import tempfile
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
from http.client import HTTPConnection, HTTPException
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
import traceback
|
|
10
|
+
from contextlib import closing
|
|
11
|
+
from typing import Optional
|
|
12
|
+
import argparse
|
|
13
|
+
|
|
14
|
+
WEBSERVER_HOST = "localhost"
|
|
15
|
+
WEBSERVER_ENDPOINT = "/api/provenance/call"
|
|
16
|
+
PORT_FILE_SUFFIX = "-provenance-port.txt"
|
|
17
|
+
|
|
18
|
+
class ProvenanceHookError(RuntimeError):
|
|
19
|
+
pass
|
|
20
|
+
|
|
21
|
+
def http_request(method, host, port, location, *, body: Optional[bytes] = None, headers={}, timeout=None, wait_for_response=False) -> bytes:
|
|
22
|
+
with closing(HTTPConnection(host, port, timeout=timeout)) as connection:
|
|
23
|
+
connection.request(method, location, body=body, headers=headers)
|
|
24
|
+
if wait_for_response:
|
|
25
|
+
response = connection.getresponse()
|
|
26
|
+
responseText = response.read()
|
|
27
|
+
|
|
28
|
+
def get_server_port():
|
|
29
|
+
claude_root = os.getenv("CLAUDE_PROJECT_DIR")
|
|
30
|
+
path_hash = hashlib.md5(claude_root.encode('utf-8')).hexdigest()
|
|
31
|
+
port_file = Path(tempfile.gettempdir()) / (path_hash + PORT_FILE_SUFFIX)
|
|
32
|
+
|
|
33
|
+
return int(port_file.read_text("utf-8").strip())
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def send_diff_to_webserver(file_path, timestamp_ms, wait_for_response):
|
|
37
|
+
try:
|
|
38
|
+
port = get_server_port()
|
|
39
|
+
except FileNotFoundError as e:
|
|
40
|
+
raise ProvenanceHookError(
|
|
41
|
+
f"Could not determine API port: {e.filename} does not exist") from e
|
|
42
|
+
except Exception as e:
|
|
43
|
+
raise ProvenanceHookError("Could not determine API port") from e
|
|
44
|
+
|
|
45
|
+
url = f"http://{WEBSERVER_HOST}:{port}{WEBSERVER_ENDPOINT}"
|
|
46
|
+
|
|
47
|
+
try:
|
|
48
|
+
payload = {"file_path": file_path, "timestamp": timestamp_ms}
|
|
49
|
+
return http_request(
|
|
50
|
+
"POST",
|
|
51
|
+
WEBSERVER_HOST,
|
|
52
|
+
port=port,
|
|
53
|
+
location=WEBSERVER_ENDPOINT,
|
|
54
|
+
body=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
|
55
|
+
headers={'Content-Type': 'application/json'},
|
|
56
|
+
timeout=0.5,
|
|
57
|
+
wait_for_response=wait_for_response
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
except (HTTPException, OSError, ConnectionError) as e:
|
|
61
|
+
raise ProvenanceHookError(
|
|
62
|
+
f"Network error while sending diff to {url}") from e
|
|
63
|
+
except Exception as e:
|
|
64
|
+
raise ProvenanceHookError(
|
|
65
|
+
f"Unknown error while sending diff to {url}") from e
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def extract_file_path(tool_name, tool_input):
|
|
69
|
+
if tool_name in ["Write", "Edit", "MultiEdit"]:
|
|
70
|
+
return tool_input.get('file_path', 'unknown')
|
|
71
|
+
if tool_name == "NotebookEdit":
|
|
72
|
+
return tool_input.get('notebook_path', 'unknown')
|
|
73
|
+
return 'unknown'
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def excepthook(type, value, traceback_):
|
|
77
|
+
traceback.print_exception(type, value, traceback_, file=sys.stderr)
|
|
78
|
+
sys.exit(1)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def main():
|
|
82
|
+
data = json.load(sys.stdin)
|
|
83
|
+
tool_name = data.get('tool_name', 'unknown')
|
|
84
|
+
|
|
85
|
+
p = argparse.ArgumentParser()
|
|
86
|
+
p.add_argument("--wait_for_response", default=False)
|
|
87
|
+
args = p.parse_args()
|
|
88
|
+
|
|
89
|
+
modification_tools = [
|
|
90
|
+
"Write", "Edit", "MultiEdit", "NotebookEdit"
|
|
91
|
+
]
|
|
92
|
+
|
|
93
|
+
if tool_name in modification_tools:
|
|
94
|
+
tool_input = data.get('tool_input', {})
|
|
95
|
+
file_path = extract_file_path(tool_name, tool_input)
|
|
96
|
+
if file_path:
|
|
97
|
+
timestamp_ms = int(datetime.now(timezone.utc).timestamp() * 1000)
|
|
98
|
+
send_diff_to_webserver(file_path, timestamp_ms, args.wait_for_response)
|
|
99
|
+
|
|
100
|
+
if __name__ == "__main__":
|
|
101
|
+
sys.excepthook = excepthook
|
|
102
|
+
sys.exit(main())
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
{
|
|
2
|
+
"permissions": {
|
|
3
|
+
"allow": [
|
|
4
|
+
"Bash(curl -s --max-time 30 https://opendata.dwd.de/weather/nwp/)",
|
|
5
|
+
"Bash(curl -s --max-time 30 https://opendata.dwd.de/weather/nwp/v1/)",
|
|
6
|
+
"Bash(curl -s --max-time 30 https://opendata.dwd.de/weather/nwp/v1/m/)",
|
|
7
|
+
"Bash(curl -s --max-time 40 https://opendata.dwd.de/weather/nwp/v1/m/icon-eu/p/)",
|
|
8
|
+
"Bash(curl -s --max-time 30 https://opendata.dwd.de/weather/nwp/v1/ -I)",
|
|
9
|
+
"Bash(curl -s --max-time 30 https://opendata.dwd.de/test/)",
|
|
10
|
+
"Bash(grep -v '^\\\\s*$')",
|
|
11
|
+
"Bash(curl -s --max-time 30 https://opendata.dwd.de/weather/)",
|
|
12
|
+
"WebFetch(domain:www.dwd.de)",
|
|
13
|
+
"WebSearch",
|
|
14
|
+
"WebFetch(domain:opendata.dwd.de)",
|
|
15
|
+
"Bash(curl -s --max-time 120 -r 0-400000 https://opendata.dwd.de/weather/nwp/content.log.bz2 -o clog.part)",
|
|
16
|
+
"Bash(bzip2 -dc clog.part)",
|
|
17
|
+
"mcp__claude_ai_Claude_Docs__guide",
|
|
18
|
+
"Bash(PYTHONPATH=src python3 -c ' *)",
|
|
19
|
+
"Read(//media/root/Data/conda/envs/**)",
|
|
20
|
+
"Bash(python3 -c \"import httpx\")",
|
|
21
|
+
"Bash(pip list *)",
|
|
22
|
+
"Bash(echo \"exit=$?\")",
|
|
23
|
+
"Bash(python3 -c ' *)",
|
|
24
|
+
"Bash(python3 -m pip install -q -e \".[dev]\")",
|
|
25
|
+
"Bash(python3 -c \"import httpx,pytest,dwdopen;print\\('ok', httpx.__version__, pytest.__version__\\)\")",
|
|
26
|
+
"Bash(conda install *)",
|
|
27
|
+
"Bash(python3 -m pip install -q -e .)",
|
|
28
|
+
"Bash(python3 -c \"import httpx,pytest,dwdopen;print\\('ok',httpx.__version__,pytest.__version__\\)\")",
|
|
29
|
+
"Bash(python3 -m pytest -q -m \"not live\")",
|
|
30
|
+
"Bash(python3 -m pytest -q -m live)",
|
|
31
|
+
"Bash(python3 -m ruff check src tests)",
|
|
32
|
+
"Bash(python3 -m mypy src)",
|
|
33
|
+
"Bash(python3 -m pip install -q ruff mypy)",
|
|
34
|
+
"Bash(python3 -m ruff check src tests --output-format=concise)",
|
|
35
|
+
"Bash(python3 -m ruff check src/dwdopen/_fileserver/catalogue.py --output-format=concise)",
|
|
36
|
+
"Bash(python3 -m pytest -q)",
|
|
37
|
+
"Bash(python3 -m ruff check src/dwdopen/_fileserver/catalogue.py src/dwdopen/nwp/catalogue.py --output-format=concise)",
|
|
38
|
+
"Bash(echo \"rc=$?\")",
|
|
39
|
+
"Bash(python3 -m ruff check src/dwdopen/_fileserver/catalogue.py src/dwdopen/nwp/catalogue.py tests/unit/test_catalogue.py --output-format=concise)",
|
|
40
|
+
"Bash(python3 -m ruff check tests/unit/test_catalogue.py --fix --output-format=concise)",
|
|
41
|
+
"Bash(python3 -m ruff check src/dwdopen/_fileserver/catalogue.py tests/unit/test_catalogue.py --output-format=concise)",
|
|
42
|
+
"Bash(git mv *)",
|
|
43
|
+
"Bash(python3 -m mypy src/dwdopen/client.py)",
|
|
44
|
+
"Bash(python3 -m ruff check src/dwdopen/client.py src/dwdopen/_fileserver/traversal.py tests/unit/test_traversal.py --output-format=concise)",
|
|
45
|
+
"Bash(python3 *)",
|
|
46
|
+
"Bash(sed -n '/def parameter\\(/,/raise NotImplementedError/p' src/dwdopen/nwp/model.py)",
|
|
47
|
+
"Bash(sed -n '/TODO levels and ensembles NYI/,/^ return Query/p' src/dwdopen/nwp/model.py)",
|
|
48
|
+
"Bash(grib_ls -p shortName,typeOfFirstFixedSurface,level lv/press.grib2)",
|
|
49
|
+
"Bash(cdo -s sinfon lv/press.grib2)",
|
|
50
|
+
"Bash(sed -i 's/^def _expand_level_cadence\\(selector: Every, kind: LevelType\\) -> list\\\\[Decimal\\\\]:$/def _expand_level_cadence\\(\\\\n selector: Every[LevelScalar], kind: LevelType\\\\n\\) -> list[Decimal]:/' src/dwdopen/nwp/query.py)",
|
|
51
|
+
"Bash(sed -i 's/^def _expand_cadence\\(selector: Every\\) -> list\\\\[timedelta\\\\]:$/def _expand_cadence\\(selector: Every[StepScalar]\\) -> list[timedelta]:/' src/dwdopen/nwp/query.py)",
|
|
52
|
+
"Bash(sed -i 's/^ LevelSelector,$/ LevelScalar,\\\\n LevelSelector,/' src/dwdopen/nwp/query.py)",
|
|
53
|
+
"Bash(sed -i 's/^ StepSelector,$/ StepScalar,\\\\n StepSelector,/' src/dwdopen/nwp/query.py)"
|
|
54
|
+
]
|
|
55
|
+
},
|
|
56
|
+
"hooks": {
|
|
57
|
+
"PostToolUse": [
|
|
58
|
+
{
|
|
59
|
+
"matcher": "Write|Edit|MultiEdit|NotebookEdit",
|
|
60
|
+
"hooks": [
|
|
61
|
+
{
|
|
62
|
+
"type": "command",
|
|
63
|
+
"command": "python3 \"$CLAUDE_PROJECT_DIR\"/.claude/hooks/post-tool-call.py"
|
|
64
|
+
}
|
|
65
|
+
]
|
|
66
|
+
}
|
|
67
|
+
]
|
|
68
|
+
}
|
|
69
|
+
}
|
dwdopen-0.2.1/.gitignore
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
Versions follow [semantic versioning](https://semver.org/).
|
|
4
|
+
|
|
5
|
+
## 0.2.1
|
|
6
|
+
|
|
7
|
+
- Lowered floor to Python 3.11.
|
|
8
|
+
- Packaging: added project URLs.
|
|
9
|
+
- Simplified some code and sugar.
|
|
10
|
+
- Moved from pre-alpha to alpha, added this changelog.
|
|
11
|
+
|
|
12
|
+
## 0.2.0
|
|
13
|
+
|
|
14
|
+
- Add support to download ensemble members. `members=` on `select()`, taking a number,
|
|
15
|
+
a list, `Between`, `Every` or `"all"`.
|
|
16
|
+
- **`combine="member"`** writes one combined file per member, each named after it.
|
|
17
|
+
- Plans report members in their repr and in generated file names.
|
|
18
|
+
|
|
19
|
+
## 0.1.5
|
|
20
|
+
|
|
21
|
+
- Already-downloaded files are skipped. A file counts as done when its size matches
|
|
22
|
+
the catalogue and it starts with `GRIB` and ends with `7777`.
|
|
23
|
+
- Generated file names end in a hash of the plan, which is what makes the check possible.
|
|
24
|
+
- `DownloadResult.assets_skipped` reports on how much has been skipped and how much needs
|
|
25
|
+
to transfer.
|
|
26
|
+
- Interrupted downloads clean up the temporary folder(s).
|
|
27
|
+
|
|
28
|
+
## 0.1.4
|
|
29
|
+
|
|
30
|
+
- Added support for time-invariant fields.
|
|
31
|
+
|
|
32
|
+
## 0.1.3
|
|
33
|
+
|
|
34
|
+
- Generated file names. Give `combine="all"` a directory and it names the file after the
|
|
35
|
+
model, run, selection and step range.
|
|
36
|
+
- `ResolvedRequest` and `Asset` print a readable summary instead of listing every asset.
|
|
37
|
+
|
|
38
|
+
## 0.1.2
|
|
39
|
+
|
|
40
|
+
- Model and parameter names are matched ignoring case, so `t_2m` and `T_2M` both work.
|
|
41
|
+
- `hours()` and `minutes()` convenience functions from plain numbers.
|
|
42
|
+
- A bare number as a step is refused with a message, as it might be ambiguous.
|
|
43
|
+
|
|
44
|
+
## 0.1.1
|
|
45
|
+
|
|
46
|
+
- Vertical levels: `level_type=` and `levels=`, in hPa for pressure, indices for model
|
|
47
|
+
levels and metres for soil.
|
|
48
|
+
- `AmbiguousSelectionError` when a parameter exists on several level types and none was
|
|
49
|
+
named.
|
|
50
|
+
- Level listings are fetched concurrently.
|
|
51
|
+
|
|
52
|
+
## 0.1.0
|
|
53
|
+
|
|
54
|
+
- First release. Model and parameter discovery, step selection, run resolution, and
|
|
55
|
+
parallel downloads with atomic writes.
|
|
56
|
+
- Messages in a combined file are sorted time-major.
|
dwdopen-0.2.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Christoph Fischer
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
dwdopen-0.2.1/PKG-INFO
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: dwdopen
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Summary: Typed Python access to DWD Open Data numerical weather prediction (ICON).
|
|
5
|
+
Project-URL: Homepage, https://github.com/MuffinCompiler/dwdopen
|
|
6
|
+
Project-URL: Repository, https://github.com/MuffinCompiler/dwdopen
|
|
7
|
+
Project-URL: Issues, https://github.com/MuffinCompiler/dwdopen/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/MuffinCompiler/dwdopen/blob/master/CHANGELOG.md
|
|
9
|
+
Author: Christoph Fischer
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: dwd,grib,icon,nwp,opendata,weather
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
|
|
18
|
+
Classifier: Typing :: Typed
|
|
19
|
+
Requires-Python: >=3.11
|
|
20
|
+
Requires-Dist: httpx>=0.27
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Requires-Dist: mypy>=1.11; extra == 'dev'
|
|
23
|
+
Requires-Dist: pytest-cov; extra == 'dev'
|
|
24
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
25
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
# dwdopen
|
|
29
|
+
|
|
30
|
+
Python library to access [DWD Open Data](https://opendata.dwd.de/) numerical weather
|
|
31
|
+
prediction (NWP) data: discover what data is currently offered, define what you want,
|
|
32
|
+
and download it.
|
|
33
|
+
|
|
34
|
+
> Still in "pre-alpha", not everything works yet.
|
|
35
|
+
> See [Not yet implemented](#not-yet-implemented).
|
|
36
|
+
|
|
37
|
+
## Install
|
|
38
|
+
|
|
39
|
+
Requires **Python 3.12+**.
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
pip install git+https://github.com/MuffinCompiler/dwdopen@v0.1.0
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Quickstart
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
from dwdopen import DWD
|
|
49
|
+
|
|
50
|
+
with DWD() as dwd:
|
|
51
|
+
icon_eu = dwd.nwp.model("icon-eu")
|
|
52
|
+
|
|
53
|
+
query = icon_eu.select(parameters=["T_2M", "PMSL"], steps="6h")
|
|
54
|
+
query.download("forecast.grib2")
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Nothing touches the network until a call needs availability information.
|
|
58
|
+
|
|
59
|
+
### Investigate your request before you download
|
|
60
|
+
|
|
61
|
+
`resolve()` freezes a query against one run and hands back a plan you can inspect before
|
|
62
|
+
any download happens:
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
plan = icon_eu.select(parameters="U", level_type="pressure").resolve()
|
|
66
|
+
print(plan)
|
|
67
|
+
# ResolvedRequest(run=2026-09-23T06:00:00+00:00, 1860 assets, 1.5 GB, U,
|
|
68
|
+
# on pressure (100), 20 levels, steps 0h..120h)
|
|
69
|
+
|
|
70
|
+
print(plan.assets[0])
|
|
71
|
+
# Asset(U, 50 hPa, 0h, 723.4 KB)
|
|
72
|
+
|
|
73
|
+
plan.download("u.grib2")
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
A plan never switches to a newer run later, so what you inspected is what you get.
|
|
77
|
+
|
|
78
|
+
### Discovery
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
dwd.nwp.models() # every model DWD currently publishes
|
|
82
|
+
icon_eu.parameters() # every parameter of one model
|
|
83
|
+
icon_eu.parameter("T").level_types # (pressure (100), model (150))
|
|
84
|
+
icon_eu.levels("T", "pressure") # available level values
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Everything comes from the live catalogue, so a parameter
|
|
88
|
+
DWD adds should show up without a new dwdopen release.
|
|
89
|
+
|
|
90
|
+
## Selecting
|
|
91
|
+
|
|
92
|
+
### Forecast steps
|
|
93
|
+
|
|
94
|
+
Steps are **durations**, ICON-D2 publishes precipitation every
|
|
95
|
+
15 minutes and ICON-D2-RUC every 5.
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
from dwdopen import Between, Every, hours, minutes
|
|
99
|
+
|
|
100
|
+
steps="6h" # exactly this step
|
|
101
|
+
steps=["0h", "3h", "6h"] # exactly these
|
|
102
|
+
steps=Every("0h", "48h", "3h") # interval, inclusive at both ends
|
|
103
|
+
steps=Between("0h", "48h") # whatever exists in the interval
|
|
104
|
+
steps="all" # everything published
|
|
105
|
+
|
|
106
|
+
steps=hours(0, 6, 12, 18) # plain numbers, if that is what you have
|
|
107
|
+
steps=hours(range(0, 121, 3)) # or any iterable of them
|
|
108
|
+
steps=minutes(0, 15, 30) # for the sub-hourly models
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
`hours()` and `minutes()` turn a single number into a single duration, so they compose
|
|
112
|
+
with the others too: `Every(hours(0), hours(48), hours(3))`.
|
|
113
|
+
|
|
114
|
+
`Every` names exact steps, so a missing one makes the request incomplete. `Between` and
|
|
115
|
+
`"all"` ask for whatever is there and can never be incomplete.
|
|
116
|
+
|
|
117
|
+
Note that DWD publishes data incrementally on the server, so a run might just be halfway
|
|
118
|
+
available. If you use `Between`, you might just get what's already available, not the full
|
|
119
|
+
data.
|
|
120
|
+
|
|
121
|
+
### Ensemble members
|
|
122
|
+
|
|
123
|
+
The `-eps` models publish each member separately. Members are plain integers; how many
|
|
124
|
+
there are differs per model (40 for ICON-EPS and ICON-EU-EPS, 20 for ICON-D2-EPS, 10 for
|
|
125
|
+
ICON-ART-EPS):
|
|
126
|
+
|
|
127
|
+
```python
|
|
128
|
+
eps.select(parameters="T_2M", members=[1, 2, 5])
|
|
129
|
+
eps.select(parameters="T_2M", members=Between(1, 10))
|
|
130
|
+
eps.select(parameters="T_2M") # every member
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Leaving `members` out takes all of them, the same as levels and steps.
|
|
134
|
+
Ensembles publish a reduced level set: `icon-eu-eps` `T` has 3 pressure levels
|
|
135
|
+
and 3 model levels, against 20 and 74 for deterministic ICON-EU. Check the output of
|
|
136
|
+
`model.levels()` to see what is actually there.
|
|
137
|
+
|
|
138
|
+
### Vertical levels
|
|
139
|
+
|
|
140
|
+
```python
|
|
141
|
+
icon_eu.select(parameters="T", level_type="pressure", levels=[850, 500])
|
|
142
|
+
icon_eu.select(parameters="T", level_type="model", levels=Between(60, 74))
|
|
143
|
+
icon_eu.select(parameters="T_SO", level_type="soil", levels="all")
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
`level_type` may be left out only while the selection is unambiguous. `T` is available on
|
|
147
|
+
both pressure and model levels, so omitting it raises `AmbiguousSelectionError` rather
|
|
148
|
+
than guessing. A 2-D field such as `T_2M` needs no level type at all. Time-invariant fields
|
|
149
|
+
need no level type either. These are published under every run with a single step.
|
|
150
|
+
`HHL` is the exception, being model half-level heights:
|
|
151
|
+
|
|
152
|
+
```python
|
|
153
|
+
icon_d2.select(parameters="HSURF", steps="0h") # no level type
|
|
154
|
+
icon_d2.select(parameters="HHL", level_type="model", levels="all")
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
## Downloading
|
|
158
|
+
|
|
159
|
+
```python
|
|
160
|
+
query.download("forecast.grib2") # one combined file
|
|
161
|
+
query.download("members/", combine="member") # one file per ensemble member
|
|
162
|
+
query.download("forecast/", combine="none") # one file per message
|
|
163
|
+
query.download("f.grib2", temp_dir="/scratch") # partial downloads before combining elsewhere
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
`combine="all"` concatenates the messages into a single GRIB2 file, which is valid
|
|
167
|
+
because GRIB2 messages are self-delimiting. You can also provide just a directory in
|
|
168
|
+
this case and the name is generated automatically.
|
|
169
|
+
|
|
170
|
+
```python
|
|
171
|
+
plan = query.resolve()
|
|
172
|
+
plan.suggested_name() # 'icon-eu_2026-09-24T0600_PMSL+T_2M_0h-24h.grib2'
|
|
173
|
+
plan.download("/home/weather/") # writes that name into the directory
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
Messages are sorted time-major: every field of one forecast step together, steps
|
|
177
|
+
ascending. Some programs like CDO require grib files to be sorted by time to work
|
|
178
|
+
correctly.
|
|
179
|
+
|
|
180
|
+
Downloads run concurrently, are written to a temporary name and renamed into place, so a
|
|
181
|
+
partial file is never mistaken for a finished one. Several processes may write the same
|
|
182
|
+
directory at once.
|
|
183
|
+
|
|
184
|
+
Already downloaded files are skipped, a second run of the same selection transfers nothing:
|
|
185
|
+
A file counts as done when its size matches the catalogue and it starts with `GRIB`
|
|
186
|
+
and ends with `7777`. For `combine="all"` this works when the name was generated, because
|
|
187
|
+
the generated name ends in a hash of the plan and so describes that exact selection.
|
|
188
|
+
|
|
189
|
+
`DownloadResult.assets_skipped` says how many were already there, which is worth
|
|
190
|
+
checking in a scheduled job. Nothing is kept between runs. A partial download is discarded,
|
|
191
|
+
so an interrupted job never leaves files behind.
|
|
192
|
+
|
|
193
|
+
`temp_dir` must be on the same filesystem as the destination, because publishing a
|
|
194
|
+
finished file is a rename and a rename cannot cross filesystems.
|
|
195
|
+
|
|
196
|
+
## Logging
|
|
197
|
+
|
|
198
|
+
The library logs through the standard `dwdopen` logger. It never prompts or reads from
|
|
199
|
+
stdin as the library is meant to run unattended.
|
|
200
|
+
|
|
201
|
+
```python
|
|
202
|
+
import logging
|
|
203
|
+
logging.basicConfig(level=logging.INFO)
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
`INFO` reports what is about to be downloaded and how large it is. `WARNING` covers
|
|
207
|
+
retries and mixed level types.
|
|
208
|
+
|
|
209
|
+
## Not yet implemented
|
|
210
|
+
|
|
211
|
+
- [ ] Resume and skip existing files. A failed run currently re-downloads from scratch.
|
|
212
|
+
- [ ] Ensemble members** (`-eps` models, the `e/<NN>/` path segment).
|
|
213
|
+
- [ ] `combine="member"` and `combine="parameter"`.
|
|
214
|
+
- [ ] Listing cache. Every call re-reads the catalogue today.
|
|
215
|
+
- [ ] Feedback of download progress (bar, text, visual?)
|
|
216
|
+
- [ ] ICON-ART wavelengths (the `wvl1` segment).
|
|
217
|
+
- [ ] A command-line interface.
|
|
218
|
+
- [ ] Automatic parameter choice for `Model.latest_run()`, which needs
|
|
219
|
+
`parameter=` at the moment.
|
|
220
|
+
|
|
221
|
+
## License
|
|
222
|
+
|
|
223
|
+
MIT. See [LICENSE](LICENSE).
|
|
224
|
+
|
|
225
|
+
Forecast data itself is provided by Deutscher Wetterdienst (DWD) under its own
|
|
226
|
+
[terms of use](https://www.dwd.de/EN/service/copyright/copyright_node.html).
|
dwdopen-0.2.1/README.md
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
# dwdopen
|
|
2
|
+
|
|
3
|
+
Python library to access [DWD Open Data](https://opendata.dwd.de/) numerical weather
|
|
4
|
+
prediction (NWP) data: discover what data is currently offered, define what you want,
|
|
5
|
+
and download it.
|
|
6
|
+
|
|
7
|
+
> Still in "pre-alpha", not everything works yet.
|
|
8
|
+
> See [Not yet implemented](#not-yet-implemented).
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
Requires **Python 3.12+**.
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
pip install git+https://github.com/MuffinCompiler/dwdopen@v0.1.0
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Quickstart
|
|
19
|
+
|
|
20
|
+
```python
|
|
21
|
+
from dwdopen import DWD
|
|
22
|
+
|
|
23
|
+
with DWD() as dwd:
|
|
24
|
+
icon_eu = dwd.nwp.model("icon-eu")
|
|
25
|
+
|
|
26
|
+
query = icon_eu.select(parameters=["T_2M", "PMSL"], steps="6h")
|
|
27
|
+
query.download("forecast.grib2")
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Nothing touches the network until a call needs availability information.
|
|
31
|
+
|
|
32
|
+
### Investigate your request before you download
|
|
33
|
+
|
|
34
|
+
`resolve()` freezes a query against one run and hands back a plan you can inspect before
|
|
35
|
+
any download happens:
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
plan = icon_eu.select(parameters="U", level_type="pressure").resolve()
|
|
39
|
+
print(plan)
|
|
40
|
+
# ResolvedRequest(run=2026-09-23T06:00:00+00:00, 1860 assets, 1.5 GB, U,
|
|
41
|
+
# on pressure (100), 20 levels, steps 0h..120h)
|
|
42
|
+
|
|
43
|
+
print(plan.assets[0])
|
|
44
|
+
# Asset(U, 50 hPa, 0h, 723.4 KB)
|
|
45
|
+
|
|
46
|
+
plan.download("u.grib2")
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
A plan never switches to a newer run later, so what you inspected is what you get.
|
|
50
|
+
|
|
51
|
+
### Discovery
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
dwd.nwp.models() # every model DWD currently publishes
|
|
55
|
+
icon_eu.parameters() # every parameter of one model
|
|
56
|
+
icon_eu.parameter("T").level_types # (pressure (100), model (150))
|
|
57
|
+
icon_eu.levels("T", "pressure") # available level values
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Everything comes from the live catalogue, so a parameter
|
|
61
|
+
DWD adds should show up without a new dwdopen release.
|
|
62
|
+
|
|
63
|
+
## Selecting
|
|
64
|
+
|
|
65
|
+
### Forecast steps
|
|
66
|
+
|
|
67
|
+
Steps are **durations**, ICON-D2 publishes precipitation every
|
|
68
|
+
15 minutes and ICON-D2-RUC every 5.
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
from dwdopen import Between, Every, hours, minutes
|
|
72
|
+
|
|
73
|
+
steps="6h" # exactly this step
|
|
74
|
+
steps=["0h", "3h", "6h"] # exactly these
|
|
75
|
+
steps=Every("0h", "48h", "3h") # interval, inclusive at both ends
|
|
76
|
+
steps=Between("0h", "48h") # whatever exists in the interval
|
|
77
|
+
steps="all" # everything published
|
|
78
|
+
|
|
79
|
+
steps=hours(0, 6, 12, 18) # plain numbers, if that is what you have
|
|
80
|
+
steps=hours(range(0, 121, 3)) # or any iterable of them
|
|
81
|
+
steps=minutes(0, 15, 30) # for the sub-hourly models
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
`hours()` and `minutes()` turn a single number into a single duration, so they compose
|
|
85
|
+
with the others too: `Every(hours(0), hours(48), hours(3))`.
|
|
86
|
+
|
|
87
|
+
`Every` names exact steps, so a missing one makes the request incomplete. `Between` and
|
|
88
|
+
`"all"` ask for whatever is there and can never be incomplete.
|
|
89
|
+
|
|
90
|
+
Note that DWD publishes data incrementally on the server, so a run might just be halfway
|
|
91
|
+
available. If you use `Between`, you might just get what's already available, not the full
|
|
92
|
+
data.
|
|
93
|
+
|
|
94
|
+
### Ensemble members
|
|
95
|
+
|
|
96
|
+
The `-eps` models publish each member separately. Members are plain integers; how many
|
|
97
|
+
there are differs per model (40 for ICON-EPS and ICON-EU-EPS, 20 for ICON-D2-EPS, 10 for
|
|
98
|
+
ICON-ART-EPS):
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
eps.select(parameters="T_2M", members=[1, 2, 5])
|
|
102
|
+
eps.select(parameters="T_2M", members=Between(1, 10))
|
|
103
|
+
eps.select(parameters="T_2M") # every member
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
Leaving `members` out takes all of them, the same as levels and steps.
|
|
107
|
+
Ensembles publish a reduced level set: `icon-eu-eps` `T` has 3 pressure levels
|
|
108
|
+
and 3 model levels, against 20 and 74 for deterministic ICON-EU. Check the output of
|
|
109
|
+
`model.levels()` to see what is actually there.
|
|
110
|
+
|
|
111
|
+
### Vertical levels
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
icon_eu.select(parameters="T", level_type="pressure", levels=[850, 500])
|
|
115
|
+
icon_eu.select(parameters="T", level_type="model", levels=Between(60, 74))
|
|
116
|
+
icon_eu.select(parameters="T_SO", level_type="soil", levels="all")
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
`level_type` may be left out only while the selection is unambiguous. `T` is available on
|
|
120
|
+
both pressure and model levels, so omitting it raises `AmbiguousSelectionError` rather
|
|
121
|
+
than guessing. A 2-D field such as `T_2M` needs no level type at all. Time-invariant fields
|
|
122
|
+
need no level type either. These are published under every run with a single step.
|
|
123
|
+
`HHL` is the exception, being model half-level heights:
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
icon_d2.select(parameters="HSURF", steps="0h") # no level type
|
|
127
|
+
icon_d2.select(parameters="HHL", level_type="model", levels="all")
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
## Downloading
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
query.download("forecast.grib2") # one combined file
|
|
134
|
+
query.download("members/", combine="member") # one file per ensemble member
|
|
135
|
+
query.download("forecast/", combine="none") # one file per message
|
|
136
|
+
query.download("f.grib2", temp_dir="/scratch") # partial downloads before combining elsewhere
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
`combine="all"` concatenates the messages into a single GRIB2 file, which is valid
|
|
140
|
+
because GRIB2 messages are self-delimiting. You can also provide just a directory in
|
|
141
|
+
this case and the name is generated automatically.
|
|
142
|
+
|
|
143
|
+
```python
|
|
144
|
+
plan = query.resolve()
|
|
145
|
+
plan.suggested_name() # 'icon-eu_2026-09-24T0600_PMSL+T_2M_0h-24h.grib2'
|
|
146
|
+
plan.download("/home/weather/") # writes that name into the directory
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Messages are sorted time-major: every field of one forecast step together, steps
|
|
150
|
+
ascending. Some programs like CDO require grib files to be sorted by time to work
|
|
151
|
+
correctly.
|
|
152
|
+
|
|
153
|
+
Downloads run concurrently, are written to a temporary name and renamed into place, so a
|
|
154
|
+
partial file is never mistaken for a finished one. Several processes may write the same
|
|
155
|
+
directory at once.
|
|
156
|
+
|
|
157
|
+
Already downloaded files are skipped, a second run of the same selection transfers nothing:
|
|
158
|
+
A file counts as done when its size matches the catalogue and it starts with `GRIB`
|
|
159
|
+
and ends with `7777`. For `combine="all"` this works when the name was generated, because
|
|
160
|
+
the generated name ends in a hash of the plan and so describes that exact selection.
|
|
161
|
+
|
|
162
|
+
`DownloadResult.assets_skipped` says how many were already there, which is worth
|
|
163
|
+
checking in a scheduled job. Nothing is kept between runs. A partial download is discarded,
|
|
164
|
+
so an interrupted job never leaves files behind.
|
|
165
|
+
|
|
166
|
+
`temp_dir` must be on the same filesystem as the destination, because publishing a
|
|
167
|
+
finished file is a rename and a rename cannot cross filesystems.
|
|
168
|
+
|
|
169
|
+
## Logging
|
|
170
|
+
|
|
171
|
+
The library logs through the standard `dwdopen` logger. It never prompts or reads from
|
|
172
|
+
stdin as the library is meant to run unattended.
|
|
173
|
+
|
|
174
|
+
```python
|
|
175
|
+
import logging
|
|
176
|
+
logging.basicConfig(level=logging.INFO)
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
`INFO` reports what is about to be downloaded and how large it is. `WARNING` covers
|
|
180
|
+
retries and mixed level types.
|
|
181
|
+
|
|
182
|
+
## Not yet implemented
|
|
183
|
+
|
|
184
|
+
- [ ] Resume and skip existing files. A failed run currently re-downloads from scratch.
|
|
185
|
+
- [ ] Ensemble members** (`-eps` models, the `e/<NN>/` path segment).
|
|
186
|
+
- [ ] `combine="member"` and `combine="parameter"`.
|
|
187
|
+
- [ ] Listing cache. Every call re-reads the catalogue today.
|
|
188
|
+
- [ ] Feedback of download progress (bar, text, visual?)
|
|
189
|
+
- [ ] ICON-ART wavelengths (the `wvl1` segment).
|
|
190
|
+
- [ ] A command-line interface.
|
|
191
|
+
- [ ] Automatic parameter choice for `Model.latest_run()`, which needs
|
|
192
|
+
`parameter=` at the moment.
|
|
193
|
+
|
|
194
|
+
## License
|
|
195
|
+
|
|
196
|
+
MIT. See [LICENSE](LICENSE).
|
|
197
|
+
|
|
198
|
+
Forecast data itself is provided by Deutscher Wetterdienst (DWD) under its own
|
|
199
|
+
[terms of use](https://www.dwd.de/EN/service/copyright/copyright_node.html).
|