dwdopen 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. dwdopen-0.2.1/.claude/hooks/post-tool-call.py +102 -0
  2. dwdopen-0.2.1/.claude/settings.local.json +69 -0
  3. dwdopen-0.2.1/.gitignore +12 -0
  4. dwdopen-0.2.1/CHANGELOG.md +56 -0
  5. dwdopen-0.2.1/LICENSE +21 -0
  6. dwdopen-0.2.1/PKG-INFO +226 -0
  7. dwdopen-0.2.1/README.md +199 -0
  8. dwdopen-0.2.1/design/internal_api_mock.py +220 -0
  9. dwdopen-0.2.1/design/usage_examples.py +172 -0
  10. dwdopen-0.2.1/pyproject.toml +56 -0
  11. dwdopen-0.2.1/src/dwdopen/__init__.py +61 -0
  12. dwdopen-0.2.1/src/dwdopen/_fileserver/__init__.py +4 -0
  13. dwdopen-0.2.1/src/dwdopen/_fileserver/download.py +265 -0
  14. dwdopen-0.2.1/src/dwdopen/_fileserver/http.py +60 -0
  15. dwdopen-0.2.1/src/dwdopen/_fileserver/listing.py +108 -0
  16. dwdopen-0.2.1/src/dwdopen/_fileserver/naming.py +70 -0
  17. dwdopen-0.2.1/src/dwdopen/_fileserver/paths.py +48 -0
  18. dwdopen-0.2.1/src/dwdopen/_fileserver/traversal.py +404 -0
  19. dwdopen-0.2.1/src/dwdopen/client.py +164 -0
  20. dwdopen-0.2.1/src/dwdopen/exceptions.py +147 -0
  21. dwdopen-0.2.1/src/dwdopen/nwp/__init__.py +22 -0
  22. dwdopen-0.2.1/src/dwdopen/nwp/catalogue.py +125 -0
  23. dwdopen-0.2.1/src/dwdopen/nwp/durations.py +165 -0
  24. dwdopen-0.2.1/src/dwdopen/nwp/model.py +244 -0
  25. dwdopen-0.2.1/src/dwdopen/nwp/query.py +395 -0
  26. dwdopen-0.2.1/src/dwdopen/nwp/request.py +492 -0
  27. dwdopen-0.2.1/src/dwdopen/nwp/run.py +68 -0
  28. dwdopen-0.2.1/src/dwdopen/nwp/selectors.py +176 -0
  29. dwdopen-0.2.1/src/dwdopen/py.typed +0 -0
  30. dwdopen-0.2.1/testing.py +53 -0
  31. dwdopen-0.2.1/tests/integration/test_live_discovery.py +18 -0
  32. dwdopen-0.2.1/tests/integration/test_live_download.py +175 -0
  33. dwdopen-0.2.1/tests/integration/test_live_listing.py +16 -0
  34. dwdopen-0.2.1/tests/unit/test_case_insensitivity.py +84 -0
  35. dwdopen-0.2.1/tests/unit/test_download.py +379 -0
  36. dwdopen-0.2.1/tests/unit/test_durations.py +211 -0
  37. dwdopen-0.2.1/tests/unit/test_levels.py +218 -0
  38. dwdopen-0.2.1/tests/unit/test_listing.py +55 -0
  39. dwdopen-0.2.1/tests/unit/test_members.py +160 -0
  40. dwdopen-0.2.1/tests/unit/test_naming_plans.py +205 -0
  41. dwdopen-0.2.1/tests/unit/test_paths.py +63 -0
  42. dwdopen-0.2.1/tests/unit/test_repr.py +130 -0
  43. dwdopen-0.2.1/tests/unit/test_traversal.py +181 -0
@@ -0,0 +1,102 @@
1
+ import hashlib
2
+ import json
3
+ import os
4
+ import sys
5
+ import tempfile
6
+ from datetime import datetime, timezone
7
+ from http.client import HTTPConnection, HTTPException
8
+ from pathlib import Path
9
+ import traceback
10
+ from contextlib import closing
11
+ from typing import Optional
12
+ import argparse
13
+
14
+ WEBSERVER_HOST = "localhost"
15
+ WEBSERVER_ENDPOINT = "/api/provenance/call"
16
+ PORT_FILE_SUFFIX = "-provenance-port.txt"
17
+
18
+ class ProvenanceHookError(RuntimeError):
19
+ pass
20
+
21
+ def http_request(method, host, port, location, *, body: Optional[bytes] = None, headers={}, timeout=None, wait_for_response=False) -> bytes:
22
+ with closing(HTTPConnection(host, port, timeout=timeout)) as connection:
23
+ connection.request(method, location, body=body, headers=headers)
24
+ if wait_for_response:
25
+ response = connection.getresponse()
26
+ responseText = response.read()
27
+
28
+ def get_server_port():
29
+ claude_root = os.getenv("CLAUDE_PROJECT_DIR")
30
+ path_hash = hashlib.md5(claude_root.encode('utf-8')).hexdigest()
31
+ port_file = Path(tempfile.gettempdir()) / (path_hash + PORT_FILE_SUFFIX)
32
+
33
+ return int(port_file.read_text("utf-8").strip())
34
+
35
+
36
+ def send_diff_to_webserver(file_path, timestamp_ms, wait_for_response):
37
+ try:
38
+ port = get_server_port()
39
+ except FileNotFoundError as e:
40
+ raise ProvenanceHookError(
41
+ f"Could not determine API port: {e.filename} does not exist") from e
42
+ except Exception as e:
43
+ raise ProvenanceHookError("Could not determine API port") from e
44
+
45
+ url = f"http://{WEBSERVER_HOST}:{port}{WEBSERVER_ENDPOINT}"
46
+
47
+ try:
48
+ payload = {"file_path": file_path, "timestamp": timestamp_ms}
49
+ return http_request(
50
+ "POST",
51
+ WEBSERVER_HOST,
52
+ port=port,
53
+ location=WEBSERVER_ENDPOINT,
54
+ body=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
55
+ headers={'Content-Type': 'application/json'},
56
+ timeout=0.5,
57
+ wait_for_response=wait_for_response
58
+ )
59
+
60
+ except (HTTPException, OSError, ConnectionError) as e:
61
+ raise ProvenanceHookError(
62
+ f"Network error while sending diff to {url}") from e
63
+ except Exception as e:
64
+ raise ProvenanceHookError(
65
+ f"Unknown error while sending diff to {url}") from e
66
+
67
+
68
+ def extract_file_path(tool_name, tool_input):
69
+ if tool_name in ["Write", "Edit", "MultiEdit"]:
70
+ return tool_input.get('file_path', 'unknown')
71
+ if tool_name == "NotebookEdit":
72
+ return tool_input.get('notebook_path', 'unknown')
73
+ return 'unknown'
74
+
75
+
76
+ def excepthook(type, value, traceback_):
77
+ traceback.print_exception(type, value, traceback_, file=sys.stderr)
78
+ sys.exit(1)
79
+
80
+
81
+ def main():
82
+ data = json.load(sys.stdin)
83
+ tool_name = data.get('tool_name', 'unknown')
84
+
85
+ p = argparse.ArgumentParser()
86
+ p.add_argument("--wait_for_response", default=False)
87
+ args = p.parse_args()
88
+
89
+ modification_tools = [
90
+ "Write", "Edit", "MultiEdit", "NotebookEdit"
91
+ ]
92
+
93
+ if tool_name in modification_tools:
94
+ tool_input = data.get('tool_input', {})
95
+ file_path = extract_file_path(tool_name, tool_input)
96
+ if file_path:
97
+ timestamp_ms = int(datetime.now(timezone.utc).timestamp() * 1000)
98
+ send_diff_to_webserver(file_path, timestamp_ms, args.wait_for_response)
99
+
100
+ if __name__ == "__main__":
101
+ sys.excepthook = excepthook
102
+ sys.exit(main())
@@ -0,0 +1,69 @@
1
+ {
2
+ "permissions": {
3
+ "allow": [
4
+ "Bash(curl -s --max-time 30 https://opendata.dwd.de/weather/nwp/)",
5
+ "Bash(curl -s --max-time 30 https://opendata.dwd.de/weather/nwp/v1/)",
6
+ "Bash(curl -s --max-time 30 https://opendata.dwd.de/weather/nwp/v1/m/)",
7
+ "Bash(curl -s --max-time 40 https://opendata.dwd.de/weather/nwp/v1/m/icon-eu/p/)",
8
+ "Bash(curl -s --max-time 30 https://opendata.dwd.de/weather/nwp/v1/ -I)",
9
+ "Bash(curl -s --max-time 30 https://opendata.dwd.de/test/)",
10
+ "Bash(grep -v '^\\\\s*$')",
11
+ "Bash(curl -s --max-time 30 https://opendata.dwd.de/weather/)",
12
+ "WebFetch(domain:www.dwd.de)",
13
+ "WebSearch",
14
+ "WebFetch(domain:opendata.dwd.de)",
15
+ "Bash(curl -s --max-time 120 -r 0-400000 https://opendata.dwd.de/weather/nwp/content.log.bz2 -o clog.part)",
16
+ "Bash(bzip2 -dc clog.part)",
17
+ "mcp__claude_ai_Claude_Docs__guide",
18
+ "Bash(PYTHONPATH=src python3 -c ' *)",
19
+ "Read(//media/root/Data/conda/envs/**)",
20
+ "Bash(python3 -c \"import httpx\")",
21
+ "Bash(pip list *)",
22
+ "Bash(echo \"exit=$?\")",
23
+ "Bash(python3 -c ' *)",
24
+ "Bash(python3 -m pip install -q -e \".[dev]\")",
25
+ "Bash(python3 -c \"import httpx,pytest,dwdopen;print\\('ok', httpx.__version__, pytest.__version__\\)\")",
26
+ "Bash(conda install *)",
27
+ "Bash(python3 -m pip install -q -e .)",
28
+ "Bash(python3 -c \"import httpx,pytest,dwdopen;print\\('ok',httpx.__version__,pytest.__version__\\)\")",
29
+ "Bash(python3 -m pytest -q -m \"not live\")",
30
+ "Bash(python3 -m pytest -q -m live)",
31
+ "Bash(python3 -m ruff check src tests)",
32
+ "Bash(python3 -m mypy src)",
33
+ "Bash(python3 -m pip install -q ruff mypy)",
34
+ "Bash(python3 -m ruff check src tests --output-format=concise)",
35
+ "Bash(python3 -m ruff check src/dwdopen/_fileserver/catalogue.py --output-format=concise)",
36
+ "Bash(python3 -m pytest -q)",
37
+ "Bash(python3 -m ruff check src/dwdopen/_fileserver/catalogue.py src/dwdopen/nwp/catalogue.py --output-format=concise)",
38
+ "Bash(echo \"rc=$?\")",
39
+ "Bash(python3 -m ruff check src/dwdopen/_fileserver/catalogue.py src/dwdopen/nwp/catalogue.py tests/unit/test_catalogue.py --output-format=concise)",
40
+ "Bash(python3 -m ruff check tests/unit/test_catalogue.py --fix --output-format=concise)",
41
+ "Bash(python3 -m ruff check src/dwdopen/_fileserver/catalogue.py tests/unit/test_catalogue.py --output-format=concise)",
42
+ "Bash(git mv *)",
43
+ "Bash(python3 -m mypy src/dwdopen/client.py)",
44
+ "Bash(python3 -m ruff check src/dwdopen/client.py src/dwdopen/_fileserver/traversal.py tests/unit/test_traversal.py --output-format=concise)",
45
+ "Bash(python3 *)",
46
+ "Bash(sed -n '/def parameter\\(/,/raise NotImplementedError/p' src/dwdopen/nwp/model.py)",
47
+ "Bash(sed -n '/TODO levels and ensembles NYI/,/^ return Query/p' src/dwdopen/nwp/model.py)",
48
+ "Bash(grib_ls -p shortName,typeOfFirstFixedSurface,level lv/press.grib2)",
49
+ "Bash(cdo -s sinfon lv/press.grib2)",
50
+ "Bash(sed -i 's/^def _expand_level_cadence\\(selector: Every, kind: LevelType\\) -> list\\\\[Decimal\\\\]:$/def _expand_level_cadence\\(\\\\n selector: Every[LevelScalar], kind: LevelType\\\\n\\) -> list[Decimal]:/' src/dwdopen/nwp/query.py)",
51
+ "Bash(sed -i 's/^def _expand_cadence\\(selector: Every\\) -> list\\\\[timedelta\\\\]:$/def _expand_cadence\\(selector: Every[StepScalar]\\) -> list[timedelta]:/' src/dwdopen/nwp/query.py)",
52
+ "Bash(sed -i 's/^ LevelSelector,$/ LevelScalar,\\\\n LevelSelector,/' src/dwdopen/nwp/query.py)",
53
+ "Bash(sed -i 's/^ StepSelector,$/ StepScalar,\\\\n StepSelector,/' src/dwdopen/nwp/query.py)"
54
+ ]
55
+ },
56
+ "hooks": {
57
+ "PostToolUse": [
58
+ {
59
+ "matcher": "Write|Edit|MultiEdit|NotebookEdit",
60
+ "hooks": [
61
+ {
62
+ "type": "command",
63
+ "command": "python3 \"$CLAUDE_PROJECT_DIR\"/.claude/hooks/post-tool-call.py"
64
+ }
65
+ ]
66
+ }
67
+ ]
68
+ }
69
+ }
@@ -0,0 +1,12 @@
1
+ # IDEs
2
+ .idea/
3
+ .vscode/
4
+ __pycache__/
5
+
6
+ # GRIB output
7
+ *.grib2
8
+ *.grb2
9
+ dist/
10
+
11
+ build/
12
+ *.egg-info/
@@ -0,0 +1,56 @@
1
+ # Changelog
2
+
3
+ Versions follow [semantic versioning](https://semver.org/).
4
+
5
+ ## 0.2.1
6
+
7
+ - Lowered floor to Python 3.11.
8
+ - Packaging: added project URLs.
9
+ - Simplified some code and sugar.
10
+ - Moved from pre-alpha to alpha, added this changelog.
11
+
12
+ ## 0.2.0
13
+
14
+ - Add support to download ensemble members. `members=` on `select()`, taking a number,
15
+ a list, `Between`, `Every` or `"all"`.
16
+ - **`combine="member"`** writes one combined file per member, each named after it.
17
+ - Plans report members in their repr and in generated file names.
18
+
19
+ ## 0.1.5
20
+
21
+ - Already-downloaded files are skipped. A file counts as done when its size matches
22
+ the catalogue and it starts with `GRIB` and ends with `7777`.
23
+ - Generated file names end in a hash of the plan, which is what makes the check possible.
24
+ - `DownloadResult.assets_skipped` reports on how much has been skipped and how much needs
25
+ to transfer.
26
+ - Interrupted downloads clean up the temporary folder(s).
27
+
28
+ ## 0.1.4
29
+
30
+ - Added support for time-invariant fields.
31
+
32
+ ## 0.1.3
33
+
34
+ - Generated file names. Give `combine="all"` a directory and it names the file after the
35
+ model, run, selection and step range.
36
+ - `ResolvedRequest` and `Asset` print a readable summary instead of listing every asset.
37
+
38
+ ## 0.1.2
39
+
40
+ - Model and parameter names are matched ignoring case, so `t_2m` and `T_2M` both work.
41
+ - `hours()` and `minutes()` convenience functions from plain numbers.
42
+ - A bare number as a step is refused with a message, as it might be ambiguous.
43
+
44
+ ## 0.1.1
45
+
46
+ - Vertical levels: `level_type=` and `levels=`, in hPa for pressure, indices for model
47
+ levels and metres for soil.
48
+ - `AmbiguousSelectionError` when a parameter exists on several level types and none was
49
+ named.
50
+ - Level listings are fetched concurrently.
51
+
52
+ ## 0.1.0
53
+
54
+ - First release. Model and parameter discovery, step selection, run resolution, and
55
+ parallel downloads with atomic writes.
56
+ - Messages in a combined file are sorted time-major.
dwdopen-0.2.1/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Christoph Fischer
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
dwdopen-0.2.1/PKG-INFO ADDED
@@ -0,0 +1,226 @@
1
+ Metadata-Version: 2.5
2
+ Name: dwdopen
3
+ Version: 0.2.1
4
+ Summary: Typed Python access to DWD Open Data numerical weather prediction (ICON).
5
+ Project-URL: Homepage, https://github.com/MuffinCompiler/dwdopen
6
+ Project-URL: Repository, https://github.com/MuffinCompiler/dwdopen
7
+ Project-URL: Issues, https://github.com/MuffinCompiler/dwdopen/issues
8
+ Project-URL: Changelog, https://github.com/MuffinCompiler/dwdopen/blob/master/CHANGELOG.md
9
+ Author: Christoph Fischer
10
+ License-Expression: MIT
11
+ License-File: LICENSE
12
+ Keywords: dwd,grib,icon,nwp,opendata,weather
13
+ Classifier: Development Status :: 3 - Alpha
14
+ Classifier: Intended Audience :: Science/Research
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
18
+ Classifier: Typing :: Typed
19
+ Requires-Python: >=3.11
20
+ Requires-Dist: httpx>=0.27
21
+ Provides-Extra: dev
22
+ Requires-Dist: mypy>=1.11; extra == 'dev'
23
+ Requires-Dist: pytest-cov; extra == 'dev'
24
+ Requires-Dist: pytest>=8; extra == 'dev'
25
+ Requires-Dist: ruff>=0.6; extra == 'dev'
26
+ Description-Content-Type: text/markdown
27
+
28
+ # dwdopen
29
+
30
+ Python library to access [DWD Open Data](https://opendata.dwd.de/) numerical weather
31
+ prediction (NWP) data: discover what data is currently offered, define what you want,
32
+ and download it.
33
+
34
+ > Still in "pre-alpha", not everything works yet.
35
+ > See [Not yet implemented](#not-yet-implemented).
36
+
37
+ ## Install
38
+
39
+ Requires **Python 3.12+**.
40
+
41
+ ```bash
42
+ pip install git+https://github.com/MuffinCompiler/dwdopen@v0.1.0
43
+ ```
44
+
45
+ ## Quickstart
46
+
47
+ ```python
48
+ from dwdopen import DWD
49
+
50
+ with DWD() as dwd:
51
+ icon_eu = dwd.nwp.model("icon-eu")
52
+
53
+ query = icon_eu.select(parameters=["T_2M", "PMSL"], steps="6h")
54
+ query.download("forecast.grib2")
55
+ ```
56
+
57
+ Nothing touches the network until a call needs availability information.
58
+
59
+ ### Investigate your request before you download
60
+
61
+ `resolve()` freezes a query against one run and hands back a plan you can inspect before
62
+ any download happens:
63
+
64
+ ```python
65
+ plan = icon_eu.select(parameters="U", level_type="pressure").resolve()
66
+ print(plan)
67
+ # ResolvedRequest(run=2026-09-23T06:00:00+00:00, 1860 assets, 1.5 GB, U,
68
+ # on pressure (100), 20 levels, steps 0h..120h)
69
+
70
+ print(plan.assets[0])
71
+ # Asset(U, 50 hPa, 0h, 723.4 KB)
72
+
73
+ plan.download("u.grib2")
74
+ ```
75
+
76
+ A plan never switches to a newer run later, so what you inspected is what you get.
77
+
78
+ ### Discovery
79
+
80
+ ```python
81
+ dwd.nwp.models() # every model DWD currently publishes
82
+ icon_eu.parameters() # every parameter of one model
83
+ icon_eu.parameter("T").level_types # (pressure (100), model (150))
84
+ icon_eu.levels("T", "pressure") # available level values
85
+ ```
86
+
87
+ Everything comes from the live catalogue, so a parameter
88
+ DWD adds should show up without a new dwdopen release.
89
+
90
+ ## Selecting
91
+
92
+ ### Forecast steps
93
+
94
+ Steps are **durations**, ICON-D2 publishes precipitation every
95
+ 15 minutes and ICON-D2-RUC every 5.
96
+
97
+ ```python
98
+ from dwdopen import Between, Every, hours, minutes
99
+
100
+ steps="6h" # exactly this step
101
+ steps=["0h", "3h", "6h"] # exactly these
102
+ steps=Every("0h", "48h", "3h") # interval, inclusive at both ends
103
+ steps=Between("0h", "48h") # whatever exists in the interval
104
+ steps="all" # everything published
105
+
106
+ steps=hours(0, 6, 12, 18) # plain numbers, if that is what you have
107
+ steps=hours(range(0, 121, 3)) # or any iterable of them
108
+ steps=minutes(0, 15, 30) # for the sub-hourly models
109
+ ```
110
+
111
+ `hours()` and `minutes()` turn a single number into a single duration, so they compose
112
+ with the others too: `Every(hours(0), hours(48), hours(3))`.
113
+
114
+ `Every` names exact steps, so a missing one makes the request incomplete. `Between` and
115
+ `"all"` ask for whatever is there and can never be incomplete.
116
+
117
+ Note that DWD publishes data incrementally on the server, so a run might just be halfway
118
+ available. If you use `Between`, you might just get what's already available, not the full
119
+ data.
120
+
121
+ ### Ensemble members
122
+
123
+ The `-eps` models publish each member separately. Members are plain integers; how many
124
+ there are differs per model (40 for ICON-EPS and ICON-EU-EPS, 20 for ICON-D2-EPS, 10 for
125
+ ICON-ART-EPS):
126
+
127
+ ```python
128
+ eps.select(parameters="T_2M", members=[1, 2, 5])
129
+ eps.select(parameters="T_2M", members=Between(1, 10))
130
+ eps.select(parameters="T_2M") # every member
131
+ ```
132
+
133
+ Leaving `members` out takes all of them, the same as levels and steps.
134
+ Ensembles publish a reduced level set: `icon-eu-eps` `T` has 3 pressure levels
135
+ and 3 model levels, against 20 and 74 for deterministic ICON-EU. Check the output of
136
+ `model.levels()` to see what is actually there.
137
+
138
+ ### Vertical levels
139
+
140
+ ```python
141
+ icon_eu.select(parameters="T", level_type="pressure", levels=[850, 500])
142
+ icon_eu.select(parameters="T", level_type="model", levels=Between(60, 74))
143
+ icon_eu.select(parameters="T_SO", level_type="soil", levels="all")
144
+ ```
145
+
146
+ `level_type` may be left out only while the selection is unambiguous. `T` is available on
147
+ both pressure and model levels, so omitting it raises `AmbiguousSelectionError` rather
148
+ than guessing. A 2-D field such as `T_2M` needs no level type at all. Time-invariant fields
149
+ need no level type either. These are published under every run with a single step.
150
+ `HHL` is the exception, being model half-level heights:
151
+
152
+ ```python
153
+ icon_d2.select(parameters="HSURF", steps="0h") # no level type
154
+ icon_d2.select(parameters="HHL", level_type="model", levels="all")
155
+ ```
156
+
157
+ ## Downloading
158
+
159
+ ```python
160
+ query.download("forecast.grib2") # one combined file
161
+ query.download("members/", combine="member") # one file per ensemble member
162
+ query.download("forecast/", combine="none") # one file per message
163
+ query.download("f.grib2", temp_dir="/scratch") # partial downloads before combining elsewhere
164
+ ```
165
+
166
+ `combine="all"` concatenates the messages into a single GRIB2 file, which is valid
167
+ because GRIB2 messages are self-delimiting. You can also provide just a directory in
168
+ this case and the name is generated automatically.
169
+
170
+ ```python
171
+ plan = query.resolve()
172
+ plan.suggested_name() # 'icon-eu_2026-09-24T0600_PMSL+T_2M_0h-24h.grib2'
173
+ plan.download("/home/weather/") # writes that name into the directory
174
+ ```
175
+
176
+ Messages are sorted time-major: every field of one forecast step together, steps
177
+ ascending. Some programs like CDO require grib files to be sorted by time to work
178
+ correctly.
179
+
180
+ Downloads run concurrently, are written to a temporary name and renamed into place, so a
181
+ partial file is never mistaken for a finished one. Several processes may write the same
182
+ directory at once.
183
+
184
+ Already downloaded files are skipped, a second run of the same selection transfers nothing:
185
+ A file counts as done when its size matches the catalogue and it starts with `GRIB`
186
+ and ends with `7777`. For `combine="all"` this works when the name was generated, because
187
+ the generated name ends in a hash of the plan and so describes that exact selection.
188
+
189
+ `DownloadResult.assets_skipped` says how many were already there, which is worth
190
+ checking in a scheduled job. Nothing is kept between runs. A partial download is discarded,
191
+ so an interrupted job never leaves files behind.
192
+
193
+ `temp_dir` must be on the same filesystem as the destination, because publishing a
194
+ finished file is a rename and a rename cannot cross filesystems.
195
+
196
+ ## Logging
197
+
198
+ The library logs through the standard `dwdopen` logger. It never prompts or reads from
199
+ stdin as the library is meant to run unattended.
200
+
201
+ ```python
202
+ import logging
203
+ logging.basicConfig(level=logging.INFO)
204
+ ```
205
+
206
+ `INFO` reports what is about to be downloaded and how large it is. `WARNING` covers
207
+ retries and mixed level types.
208
+
209
+ ## Not yet implemented
210
+
211
+ - [ ] Resume and skip existing files. A failed run currently re-downloads from scratch.
212
+ - [ ] Ensemble members** (`-eps` models, the `e/<NN>/` path segment).
213
+ - [ ] `combine="member"` and `combine="parameter"`.
214
+ - [ ] Listing cache. Every call re-reads the catalogue today.
215
+ - [ ] Feedback of download progress (bar, text, visual?)
216
+ - [ ] ICON-ART wavelengths (the `wvl1` segment).
217
+ - [ ] A command-line interface.
218
+ - [ ] Automatic parameter choice for `Model.latest_run()`, which needs
219
+ `parameter=` at the moment.
220
+
221
+ ## License
222
+
223
+ MIT. See [LICENSE](LICENSE).
224
+
225
+ Forecast data itself is provided by Deutscher Wetterdienst (DWD) under its own
226
+ [terms of use](https://www.dwd.de/EN/service/copyright/copyright_node.html).
@@ -0,0 +1,199 @@
1
+ # dwdopen
2
+
3
+ Python library to access [DWD Open Data](https://opendata.dwd.de/) numerical weather
4
+ prediction (NWP) data: discover what data is currently offered, define what you want,
5
+ and download it.
6
+
7
+ > Still in "pre-alpha", not everything works yet.
8
+ > See [Not yet implemented](#not-yet-implemented).
9
+
10
+ ## Install
11
+
12
+ Requires **Python 3.12+**.
13
+
14
+ ```bash
15
+ pip install git+https://github.com/MuffinCompiler/dwdopen@v0.1.0
16
+ ```
17
+
18
+ ## Quickstart
19
+
20
+ ```python
21
+ from dwdopen import DWD
22
+
23
+ with DWD() as dwd:
24
+ icon_eu = dwd.nwp.model("icon-eu")
25
+
26
+ query = icon_eu.select(parameters=["T_2M", "PMSL"], steps="6h")
27
+ query.download("forecast.grib2")
28
+ ```
29
+
30
+ Nothing touches the network until a call needs availability information.
31
+
32
+ ### Investigate your request before you download
33
+
34
+ `resolve()` freezes a query against one run and hands back a plan you can inspect before
35
+ any download happens:
36
+
37
+ ```python
38
+ plan = icon_eu.select(parameters="U", level_type="pressure").resolve()
39
+ print(plan)
40
+ # ResolvedRequest(run=2026-09-23T06:00:00+00:00, 1860 assets, 1.5 GB, U,
41
+ # on pressure (100), 20 levels, steps 0h..120h)
42
+
43
+ print(plan.assets[0])
44
+ # Asset(U, 50 hPa, 0h, 723.4 KB)
45
+
46
+ plan.download("u.grib2")
47
+ ```
48
+
49
+ A plan never switches to a newer run later, so what you inspected is what you get.
50
+
51
+ ### Discovery
52
+
53
+ ```python
54
+ dwd.nwp.models() # every model DWD currently publishes
55
+ icon_eu.parameters() # every parameter of one model
56
+ icon_eu.parameter("T").level_types # (pressure (100), model (150))
57
+ icon_eu.levels("T", "pressure") # available level values
58
+ ```
59
+
60
+ Everything comes from the live catalogue, so a parameter
61
+ DWD adds should show up without a new dwdopen release.
62
+
63
+ ## Selecting
64
+
65
+ ### Forecast steps
66
+
67
+ Steps are **durations**, ICON-D2 publishes precipitation every
68
+ 15 minutes and ICON-D2-RUC every 5.
69
+
70
+ ```python
71
+ from dwdopen import Between, Every, hours, minutes
72
+
73
+ steps="6h" # exactly this step
74
+ steps=["0h", "3h", "6h"] # exactly these
75
+ steps=Every("0h", "48h", "3h") # interval, inclusive at both ends
76
+ steps=Between("0h", "48h") # whatever exists in the interval
77
+ steps="all" # everything published
78
+
79
+ steps=hours(0, 6, 12, 18) # plain numbers, if that is what you have
80
+ steps=hours(range(0, 121, 3)) # or any iterable of them
81
+ steps=minutes(0, 15, 30) # for the sub-hourly models
82
+ ```
83
+
84
+ `hours()` and `minutes()` turn a single number into a single duration, so they compose
85
+ with the others too: `Every(hours(0), hours(48), hours(3))`.
86
+
87
+ `Every` names exact steps, so a missing one makes the request incomplete. `Between` and
88
+ `"all"` ask for whatever is there and can never be incomplete.
89
+
90
+ Note that DWD publishes data incrementally on the server, so a run might just be halfway
91
+ available. If you use `Between`, you might just get what's already available, not the full
92
+ data.
93
+
94
+ ### Ensemble members
95
+
96
+ The `-eps` models publish each member separately. Members are plain integers; how many
97
+ there are differs per model (40 for ICON-EPS and ICON-EU-EPS, 20 for ICON-D2-EPS, 10 for
98
+ ICON-ART-EPS):
99
+
100
+ ```python
101
+ eps.select(parameters="T_2M", members=[1, 2, 5])
102
+ eps.select(parameters="T_2M", members=Between(1, 10))
103
+ eps.select(parameters="T_2M") # every member
104
+ ```
105
+
106
+ Leaving `members` out takes all of them, the same as levels and steps.
107
+ Ensembles publish a reduced level set: `icon-eu-eps` `T` has 3 pressure levels
108
+ and 3 model levels, against 20 and 74 for deterministic ICON-EU. Check the output of
109
+ `model.levels()` to see what is actually there.
110
+
111
+ ### Vertical levels
112
+
113
+ ```python
114
+ icon_eu.select(parameters="T", level_type="pressure", levels=[850, 500])
115
+ icon_eu.select(parameters="T", level_type="model", levels=Between(60, 74))
116
+ icon_eu.select(parameters="T_SO", level_type="soil", levels="all")
117
+ ```
118
+
119
+ `level_type` may be left out only while the selection is unambiguous. `T` is available on
120
+ both pressure and model levels, so omitting it raises `AmbiguousSelectionError` rather
121
+ than guessing. A 2-D field such as `T_2M` needs no level type at all. Time-invariant fields
122
+ need no level type either. These are published under every run with a single step.
123
+ `HHL` is the exception, being model half-level heights:
124
+
125
+ ```python
126
+ icon_d2.select(parameters="HSURF", steps="0h") # no level type
127
+ icon_d2.select(parameters="HHL", level_type="model", levels="all")
128
+ ```
129
+
130
+ ## Downloading
131
+
132
+ ```python
133
+ query.download("forecast.grib2") # one combined file
134
+ query.download("members/", combine="member") # one file per ensemble member
135
+ query.download("forecast/", combine="none") # one file per message
136
+ query.download("f.grib2", temp_dir="/scratch") # partial downloads before combining elsewhere
137
+ ```
138
+
139
+ `combine="all"` concatenates the messages into a single GRIB2 file, which is valid
140
+ because GRIB2 messages are self-delimiting. You can also provide just a directory in
141
+ this case and the name is generated automatically.
142
+
143
+ ```python
144
+ plan = query.resolve()
145
+ plan.suggested_name() # 'icon-eu_2026-09-24T0600_PMSL+T_2M_0h-24h.grib2'
146
+ plan.download("/home/weather/") # writes that name into the directory
147
+ ```
148
+
149
+ Messages are sorted time-major: every field of one forecast step together, steps
150
+ ascending. Some programs like CDO require grib files to be sorted by time to work
151
+ correctly.
152
+
153
+ Downloads run concurrently, are written to a temporary name and renamed into place, so a
154
+ partial file is never mistaken for a finished one. Several processes may write the same
155
+ directory at once.
156
+
157
+ Already downloaded files are skipped, a second run of the same selection transfers nothing:
158
+ A file counts as done when its size matches the catalogue and it starts with `GRIB`
159
+ and ends with `7777`. For `combine="all"` this works when the name was generated, because
160
+ the generated name ends in a hash of the plan and so describes that exact selection.
161
+
162
+ `DownloadResult.assets_skipped` says how many were already there, which is worth
163
+ checking in a scheduled job. Nothing is kept between runs. A partial download is discarded,
164
+ so an interrupted job never leaves files behind.
165
+
166
+ `temp_dir` must be on the same filesystem as the destination, because publishing a
167
+ finished file is a rename and a rename cannot cross filesystems.
168
+
169
+ ## Logging
170
+
171
+ The library logs through the standard `dwdopen` logger. It never prompts or reads from
172
+ stdin as the library is meant to run unattended.
173
+
174
+ ```python
175
+ import logging
176
+ logging.basicConfig(level=logging.INFO)
177
+ ```
178
+
179
+ `INFO` reports what is about to be downloaded and how large it is. `WARNING` covers
180
+ retries and mixed level types.
181
+
182
+ ## Not yet implemented
183
+
184
+ - [ ] Resume and skip existing files. A failed run currently re-downloads from scratch.
185
+ - [ ] Ensemble members** (`-eps` models, the `e/<NN>/` path segment).
186
+ - [ ] `combine="member"` and `combine="parameter"`.
187
+ - [ ] Listing cache. Every call re-reads the catalogue today.
188
+ - [ ] Feedback of download progress (bar, text, visual?)
189
+ - [ ] ICON-ART wavelengths (the `wvl1` segment).
190
+ - [ ] A command-line interface.
191
+ - [ ] Automatic parameter choice for `Model.latest_run()`, which needs
192
+ `parameter=` at the moment.
193
+
194
+ ## License
195
+
196
+ MIT. See [LICENSE](LICENSE).
197
+
198
+ Forecast data itself is provided by Deutscher Wetterdienst (DWD) under its own
199
+ [terms of use](https://www.dwd.de/EN/service/copyright/copyright_node.html).