gnsswire-grib 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gnsswire_grib-0.1.0/.gitignore +13 -0
- gnsswire_grib-0.1.0/LICENSE +21 -0
- gnsswire_grib-0.1.0/PKG-INFO +197 -0
- gnsswire_grib-0.1.0/README.md +170 -0
- gnsswire_grib-0.1.0/pyproject.toml +38 -0
- gnsswire_grib-0.1.0/src/gnsswire/grib/__init__.py +71 -0
- gnsswire_grib-0.1.0/src/gnsswire/grib/_frame.py +168 -0
- gnsswire_grib-0.1.0/src/gnsswire/grib/_message.py +172 -0
- gnsswire_grib-0.1.0/src/gnsswire/grib/_sections.py +423 -0
- gnsswire_grib-0.1.0/src/gnsswire/grib/_unpack.py +421 -0
- gnsswire_grib-0.1.0/src/gnsswire/grib/py.typed +0 -0
- gnsswire_grib-0.1.0/tests/THIRD_PARTY_NOTICES.md +26 -0
- gnsswire_grib-0.1.0/tests/data/gfs-1p00.grib2 +0 -0
- gnsswire_grib-0.1.0/tests/make_data/fetch_gfs.py +41 -0
- gnsswire_grib-0.1.0/tests/test_grib.py +755 -0
- gnsswire_grib-0.1.0/tests/test_local.py +56 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Trace Navigation LLC
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: gnsswire-grib
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: GRIB edition 2 messages (WMO FM 92): sections, templates and the packed grid point values as integers, read and written losslessly.
|
|
5
|
+
Project-URL: Homepage, https://github.com/tracenav/gnsswire
|
|
6
|
+
Project-URL: Source, https://github.com/tracenav/gnsswire/tree/main/packages/grib
|
|
7
|
+
Project-URL: Changelog, https://github.com/tracenav/gnsswire/blob/main/RELEASES.md
|
|
8
|
+
Project-URL: Issues, https://github.com/tracenav/gnsswire/issues
|
|
9
|
+
Author: Trace Navigation
|
|
10
|
+
Maintainer-email: Jacob Armstrong <hi@jacobarm.com>
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: GIS
|
|
24
|
+
Requires-Python: >=3.11
|
|
25
|
+
Requires-Dist: gnsswire-bits<0.3,>=0.2.0
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
# gnsswire-grib
|
|
29
|
+
|
|
30
|
+
GRIB edition 2 messages for Python
|
|
31
|
+
|
|
32
|
+
[](https://github.com/tracenav/gnsswire/actions/workflows/test.yml)
|
|
33
|
+
[](https://pypi.org/project/gnsswire-grib/)
|
|
34
|
+
|
|
35
|
+
gnsswire-grib reads and writes GRIB edition 2 (WMO FM 92), the binary
|
|
36
|
+
format in which numerical weather models such as NOAA's GFS and HRRR
|
|
37
|
+
publish their fields, written from WMO-No. 306, Manual on Codes, Volume
|
|
38
|
+
I.2. It gives you:
|
|
39
|
+
|
|
40
|
+
- framing: messages found in a byte stream by their section lengths, from
|
|
41
|
+
"GRIB" to "7777", with the same `parse` and `scan` as every gnsswire
|
|
42
|
+
codec;
|
|
43
|
+
- every section as a frozen dataclass of the integers on the wire, the
|
|
44
|
+
grid definition templates 3.0 (latitude/longitude) and 3.30 (Lambert
|
|
45
|
+
conformal), the product definition template 4.0 and the data
|
|
46
|
+
representation templates 5.0, 5.2 and 5.3 as field tables; other
|
|
47
|
+
templates are kept as their octets;
|
|
48
|
+
- a lossless round trip: `encode(decode(frame))` gives back the message's
|
|
49
|
+
bytes;
|
|
50
|
+
- the packed values of a field as integers, for simple packing (5.0),
|
|
51
|
+
complex packing (5.2) and complex packing with spatial differencing of
|
|
52
|
+
order 1 or 2 (5.3), with bit maps and missing value management;
|
|
53
|
+
- `scale`, the decoding formula of the specification applied to them,
|
|
54
|
+
as floats rounded once from the exact value.
|
|
55
|
+
|
|
56
|
+
It is part of [gnsswire](https://pypi.org/project/gnsswire/), codecs for
|
|
57
|
+
GNSS wire formats.
|
|
58
|
+
|
|
59
|
+
## Installation
|
|
60
|
+
|
|
61
|
+
```sh
|
|
62
|
+
$ pip install gnsswire-grib
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
gnsswire-grib requires Python 3.11+ and depends only on
|
|
66
|
+
[gnsswire-bits](https://pypi.org/project/gnsswire-bits/). `pip install
|
|
67
|
+
gnsswire` installs it with every other format.
|
|
68
|
+
|
|
69
|
+
## Usage
|
|
70
|
+
|
|
71
|
+
### Build a message
|
|
72
|
+
|
|
73
|
+
A message is its discipline (Code table 0.0) and its sections, in order:
|
|
74
|
+
identification, optionally local use, grid definition, product definition,
|
|
75
|
+
data representation, bit map, data. Here, six values on a 3 by 2 degree
|
|
76
|
+
grid, packed in 4 bits each (simple packing: data template 7.0):
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
from gnsswire import grib
|
|
80
|
+
|
|
81
|
+
ones = 2**32 - 1 # all bits set: a missing value (Regulation 92.1.4)
|
|
82
|
+
grid = grib.GridDefinition(
|
|
83
|
+
source=0, points=6, list_octets=0, list_interpretation=0, template_number=0,
|
|
84
|
+
template=grib.LatLon(
|
|
85
|
+
earth_shape=6, radius_scale=0, radius=0, major_axis_scale=0, major_axis=0,
|
|
86
|
+
minor_axis_scale=0, minor_axis=0, ni=3, nj=2, basic_angle=0, subdivisions=ones,
|
|
87
|
+
la1=50_000_000, lo1=10_000_000, resolution_flags=48, la2=49_000_000,
|
|
88
|
+
lo2=12_000_000, di=1_000_000, dj=1_000_000, scanning_mode=0)) # 10**-6 degree
|
|
89
|
+
product = grib.ProductDefinition(
|
|
90
|
+
coordinates=0, template_number=0,
|
|
91
|
+
template=grib.AnalysisForecast(
|
|
92
|
+
category=0, number=0, process=2, background_process=0, forecast_process=0,
|
|
93
|
+
cutoff_hours=0, cutoff_minutes=0, time_unit=1, forecast_time=3,
|
|
94
|
+
surface1_type=103, surface1_scale=0, surface1_value=2, # 2 m above ground
|
|
95
|
+
surface2_type=255, surface2_scale=255, surface2_value=ones))
|
|
96
|
+
representation = grib.DataRepresentation(
|
|
97
|
+
points=6, template_number=0,
|
|
98
|
+
template=grib.SimplePacking(reference=0x43898000, binary_scale=-1, # R = 275.0
|
|
99
|
+
decimal_scale=0, bits=4, original_type=0))
|
|
100
|
+
identification = grib.Identification(
|
|
101
|
+
centre=7, subcentre=0, master_version=2, local_version=1, significance=1, year=2026,
|
|
102
|
+
month=11, day=20, hour=12, minute=0, second=0, production_status=0, data_type=1)
|
|
103
|
+
data = grib.Data(bytes([0x01, 0x23, 0x45])) # the values 0, 1, 2, 3, 4, 5
|
|
104
|
+
raw = grib.encode(grib.Message(0, (identification, grid, product, representation,
|
|
105
|
+
grib.Bitmap(255), data)))
|
|
106
|
+
print(len(raw), raw[:4], raw[-4:]) # 182 b'GRIB' b'7777'
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
### Read it
|
|
110
|
+
|
|
111
|
+
`scan` finds every message in a buffer (a file read whole, or the bytes of
|
|
112
|
+
a range request), skipping what is not one; `parse` answers for one
|
|
113
|
+
offset. `decode` gives the `Message`, and `fields` each data section with
|
|
114
|
+
the sections in effect for it (sections 2 to 7, 3 to 7 or 4 to 7 may
|
|
115
|
+
repeat in a message):
|
|
116
|
+
|
|
117
|
+
```python
|
|
118
|
+
stream = b"junk" + raw + raw[:50] # for a file: open(path, "rb").read()
|
|
119
|
+
(frame,) = grib.scan(stream) # the cut-off copy is not a message
|
|
120
|
+
print(frame.offset, frame.length) # 4 182
|
|
121
|
+
message = grib.decode(frame)
|
|
122
|
+
print(grib.encode(message) == frame.data) # True
|
|
123
|
+
|
|
124
|
+
(field,) = grib.fields(message)
|
|
125
|
+
print(field.grid.template.ni, field.grid.template.nj) # 3 2
|
|
126
|
+
print(field.product.template.category, field.product.template.number) # 0 0: temperature
|
|
127
|
+
print(field.representation.template)
|
|
128
|
+
# SimplePacking(reference=1133084672, binary_scale=-1, decimal_scale=0, bits=4, original_type=0)
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
### Values
|
|
132
|
+
|
|
133
|
+
`unpack(field)` gives the integer X of every data point, in the order of
|
|
134
|
+
the grid (its scanning mode), as an `array("q")`; `missing` is `None` when
|
|
135
|
+
every point has a value, otherwise one octet per point (`PRESENT`,
|
|
136
|
+
`ABSENT` for the bit map, `MISSING_PRIMARY`, `MISSING_SECONDARY`).
|
|
137
|
+
`scale(field)` applies Regulation 92.9.4, Y = (R + X × 2^E) / 10^D, and
|
|
138
|
+
gives an `array("d")`, NaN where a point has no value. `real32(bits)`
|
|
139
|
+
gives the reference value R exactly, as a significand and a power of two:
|
|
140
|
+
|
|
141
|
+
```python
|
|
142
|
+
u = grib.unpack(field)
|
|
143
|
+
print(list(u.values), u.missing) # [0, 1, 2, 3, 4, 5] None
|
|
144
|
+
print(list(grib.scale(field))) # [275.0, 275.5, 276.0, 276.5, 277.0, 277.5]
|
|
145
|
+
print(grib.real32(field.representation.template.reference)) # (9011200, -15)
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
Each Y is the float nearest the exact value of the formula, so it does not
|
|
149
|
+
depend on the order of floating point operations. `decode` and `unpack`
|
|
150
|
+
make no floats; `scale` is the only function that does.
|
|
151
|
+
|
|
152
|
+
### Speed
|
|
153
|
+
|
|
154
|
+
Pure Python, CPython 3.14 on a 2021 laptop: a full HRRR field (Lambert
|
|
155
|
+
conformal, 1,905,141 points, template 5.3 order 2 with about 100,000
|
|
156
|
+
groups) unpacks in about 0.3 s and scales in about 0.13 s; a GFS 0.25
|
|
157
|
+
degree field (1,038,240 points) in about 0.15 s and 0.08 s. `decode`
|
|
158
|
+
itself takes well under a millisecond: the data section stays as its
|
|
159
|
+
octets until it is unpacked.
|
|
160
|
+
|
|
161
|
+
## Templates
|
|
162
|
+
|
|
163
|
+
From WMO-No. 306, Volume I.2, FM 92 GRIB edition 2 (templates version
|
|
164
|
+
23.0.0):
|
|
165
|
+
|
|
166
|
+
| Section | Template | Class |
|
|
167
|
+
|---|---|---|
|
|
168
|
+
| 1 | identification | `Identification` |
|
|
169
|
+
| 2 | local use, as octets | `LocalUse` |
|
|
170
|
+
| 3 | 3.0 latitude/longitude; 3.30 Lambert conformal | `GridDefinition` with `LatLon`, `LambertConformal` |
|
|
171
|
+
| 4 | 4.0 analysis or forecast at a horizontal level or in a layer at a point in time | `ProductDefinition` with `AnalysisForecast` |
|
|
172
|
+
| 5 | 5.0 simple packing; 5.2 complex packing; 5.3 complex packing and spatial differencing | `DataRepresentation` with `SimplePacking`, `ComplexPacking`, `SpatialDifferencing` |
|
|
173
|
+
| 6 | bit map | `Bitmap` |
|
|
174
|
+
| 7 | data, as octets; unpacked by `unpack` for 7.0, 7.2 and 7.3 | `Data` |
|
|
175
|
+
|
|
176
|
+
A section with another template keeps it as `bytes`, so every message
|
|
177
|
+
`parse` accepts decodes and encodes back. `unpack` of a field with another
|
|
178
|
+
data representation template, such as 5.40 (JPEG 2000) or 5.41 (PNG),
|
|
179
|
+
raises `Unsupported`.
|
|
180
|
+
|
|
181
|
+
## Documentation
|
|
182
|
+
|
|
183
|
+
What gnsswire-grib decodes and refuses, what its tests rest on and the
|
|
184
|
+
readings it takes where its specification is ambiguous are under
|
|
185
|
+
[Limitations](https://github.com/tracenav/gnsswire#gnsswire-grib) in the
|
|
186
|
+
[repository README](https://github.com/tracenav/gnsswire#readme), with the
|
|
187
|
+
interface every gnsswire format shares and the version policy. Release
|
|
188
|
+
notes are in [RELEASES.md](https://github.com/tracenav/gnsswire/blob/main/RELEASES.md).
|
|
189
|
+
|
|
190
|
+
## Contribute
|
|
191
|
+
|
|
192
|
+
Report bugs and request templates in the
|
|
193
|
+
[issue tracker](https://github.com/tracenav/gnsswire/issues).
|
|
194
|
+
|
|
195
|
+
## Licence
|
|
196
|
+
|
|
197
|
+
MIT.
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
# gnsswire-grib
|
|
2
|
+
|
|
3
|
+
GRIB edition 2 messages for Python
|
|
4
|
+
|
|
5
|
+
[](https://github.com/tracenav/gnsswire/actions/workflows/test.yml)
|
|
6
|
+
[](https://pypi.org/project/gnsswire-grib/)
|
|
7
|
+
|
|
8
|
+
gnsswire-grib reads and writes GRIB edition 2 (WMO FM 92), the binary
|
|
9
|
+
format in which numerical weather models such as NOAA's GFS and HRRR
|
|
10
|
+
publish their fields, written from WMO-No. 306, Manual on Codes, Volume
|
|
11
|
+
I.2. It gives you:
|
|
12
|
+
|
|
13
|
+
- framing: messages found in a byte stream by their section lengths, from
|
|
14
|
+
"GRIB" to "7777", with the same `parse` and `scan` as every gnsswire
|
|
15
|
+
codec;
|
|
16
|
+
- every section as a frozen dataclass of the integers on the wire, the
|
|
17
|
+
grid definition templates 3.0 (latitude/longitude) and 3.30 (Lambert
|
|
18
|
+
conformal), the product definition template 4.0 and the data
|
|
19
|
+
representation templates 5.0, 5.2 and 5.3 as field tables; other
|
|
20
|
+
templates are kept as their octets;
|
|
21
|
+
- a lossless round trip: `encode(decode(frame))` gives back the message's
|
|
22
|
+
bytes;
|
|
23
|
+
- the packed values of a field as integers, for simple packing (5.0),
|
|
24
|
+
complex packing (5.2) and complex packing with spatial differencing of
|
|
25
|
+
order 1 or 2 (5.3), with bit maps and missing value management;
|
|
26
|
+
- `scale`, the decoding formula of the specification applied to them,
|
|
27
|
+
as floats rounded once from the exact value.
|
|
28
|
+
|
|
29
|
+
It is part of [gnsswire](https://pypi.org/project/gnsswire/), codecs for
|
|
30
|
+
GNSS wire formats.
|
|
31
|
+
|
|
32
|
+
## Installation
|
|
33
|
+
|
|
34
|
+
```sh
|
|
35
|
+
$ pip install gnsswire-grib
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
gnsswire-grib requires Python 3.11+ and depends only on
|
|
39
|
+
[gnsswire-bits](https://pypi.org/project/gnsswire-bits/). `pip install
|
|
40
|
+
gnsswire` installs it with every other format.
|
|
41
|
+
|
|
42
|
+
## Usage
|
|
43
|
+
|
|
44
|
+
### Build a message
|
|
45
|
+
|
|
46
|
+
A message is its discipline (Code table 0.0) and its sections, in order:
|
|
47
|
+
identification, optionally local use, grid definition, product definition,
|
|
48
|
+
data representation, bit map, data. Here, six values on a 3 by 2 degree
|
|
49
|
+
grid, packed in 4 bits each (simple packing: data template 7.0):
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
from gnsswire import grib
|
|
53
|
+
|
|
54
|
+
ones = 2**32 - 1 # all bits set: a missing value (Regulation 92.1.4)
|
|
55
|
+
grid = grib.GridDefinition(
|
|
56
|
+
source=0, points=6, list_octets=0, list_interpretation=0, template_number=0,
|
|
57
|
+
template=grib.LatLon(
|
|
58
|
+
earth_shape=6, radius_scale=0, radius=0, major_axis_scale=0, major_axis=0,
|
|
59
|
+
minor_axis_scale=0, minor_axis=0, ni=3, nj=2, basic_angle=0, subdivisions=ones,
|
|
60
|
+
la1=50_000_000, lo1=10_000_000, resolution_flags=48, la2=49_000_000,
|
|
61
|
+
lo2=12_000_000, di=1_000_000, dj=1_000_000, scanning_mode=0)) # 10**-6 degree
|
|
62
|
+
product = grib.ProductDefinition(
|
|
63
|
+
coordinates=0, template_number=0,
|
|
64
|
+
template=grib.AnalysisForecast(
|
|
65
|
+
category=0, number=0, process=2, background_process=0, forecast_process=0,
|
|
66
|
+
cutoff_hours=0, cutoff_minutes=0, time_unit=1, forecast_time=3,
|
|
67
|
+
surface1_type=103, surface1_scale=0, surface1_value=2, # 2 m above ground
|
|
68
|
+
surface2_type=255, surface2_scale=255, surface2_value=ones))
|
|
69
|
+
representation = grib.DataRepresentation(
|
|
70
|
+
points=6, template_number=0,
|
|
71
|
+
template=grib.SimplePacking(reference=0x43898000, binary_scale=-1, # R = 275.0
|
|
72
|
+
decimal_scale=0, bits=4, original_type=0))
|
|
73
|
+
identification = grib.Identification(
|
|
74
|
+
centre=7, subcentre=0, master_version=2, local_version=1, significance=1, year=2026,
|
|
75
|
+
month=11, day=20, hour=12, minute=0, second=0, production_status=0, data_type=1)
|
|
76
|
+
data = grib.Data(bytes([0x01, 0x23, 0x45])) # the values 0, 1, 2, 3, 4, 5
|
|
77
|
+
raw = grib.encode(grib.Message(0, (identification, grid, product, representation,
|
|
78
|
+
grib.Bitmap(255), data)))
|
|
79
|
+
print(len(raw), raw[:4], raw[-4:]) # 182 b'GRIB' b'7777'
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
### Read it
|
|
83
|
+
|
|
84
|
+
`scan` finds every message in a buffer (a file read whole, or the bytes of
|
|
85
|
+
a range request), skipping what is not one; `parse` answers for one
|
|
86
|
+
offset. `decode` gives the `Message`, and `fields` each data section with
|
|
87
|
+
the sections in effect for it (sections 2 to 7, 3 to 7 or 4 to 7 may
|
|
88
|
+
repeat in a message):
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
stream = b"junk" + raw + raw[:50] # for a file: open(path, "rb").read()
|
|
92
|
+
(frame,) = grib.scan(stream) # the cut-off copy is not a message
|
|
93
|
+
print(frame.offset, frame.length) # 4 182
|
|
94
|
+
message = grib.decode(frame)
|
|
95
|
+
print(grib.encode(message) == frame.data) # True
|
|
96
|
+
|
|
97
|
+
(field,) = grib.fields(message)
|
|
98
|
+
print(field.grid.template.ni, field.grid.template.nj) # 3 2
|
|
99
|
+
print(field.product.template.category, field.product.template.number) # 0 0: temperature
|
|
100
|
+
print(field.representation.template)
|
|
101
|
+
# SimplePacking(reference=1133084672, binary_scale=-1, decimal_scale=0, bits=4, original_type=0)
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
### Values
|
|
105
|
+
|
|
106
|
+
`unpack(field)` gives the integer X of every data point, in the order of
|
|
107
|
+
the grid (its scanning mode), as an `array("q")`; `missing` is `None` when
|
|
108
|
+
every point has a value, otherwise one octet per point (`PRESENT`,
|
|
109
|
+
`ABSENT` for the bit map, `MISSING_PRIMARY`, `MISSING_SECONDARY`).
|
|
110
|
+
`scale(field)` applies Regulation 92.9.4, Y = (R + X × 2^E) / 10^D, and
|
|
111
|
+
gives an `array("d")`, NaN where a point has no value. `real32(bits)`
|
|
112
|
+
gives the reference value R exactly, as a significand and a power of two:
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
u = grib.unpack(field)
|
|
116
|
+
print(list(u.values), u.missing) # [0, 1, 2, 3, 4, 5] None
|
|
117
|
+
print(list(grib.scale(field))) # [275.0, 275.5, 276.0, 276.5, 277.0, 277.5]
|
|
118
|
+
print(grib.real32(field.representation.template.reference)) # (9011200, -15)
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Each Y is the float nearest the exact value of the formula, so it does not
|
|
122
|
+
depend on the order of floating point operations. `decode` and `unpack`
|
|
123
|
+
make no floats; `scale` is the only function that does.
|
|
124
|
+
|
|
125
|
+
### Speed
|
|
126
|
+
|
|
127
|
+
Pure Python, CPython 3.14 on a 2021 laptop: a full HRRR field (Lambert
|
|
128
|
+
conformal, 1,905,141 points, template 5.3 order 2 with about 100,000
|
|
129
|
+
groups) unpacks in about 0.3 s and scales in about 0.13 s; a GFS 0.25
|
|
130
|
+
degree field (1,038,240 points) in about 0.15 s and 0.08 s. `decode`
|
|
131
|
+
itself takes well under a millisecond: the data section stays as its
|
|
132
|
+
octets until it is unpacked.
|
|
133
|
+
|
|
134
|
+
## Templates
|
|
135
|
+
|
|
136
|
+
From WMO-No. 306, Volume I.2, FM 92 GRIB edition 2 (templates version
|
|
137
|
+
23.0.0):
|
|
138
|
+
|
|
139
|
+
| Section | Template | Class |
|
|
140
|
+
|---|---|---|
|
|
141
|
+
| 1 | identification | `Identification` |
|
|
142
|
+
| 2 | local use, as octets | `LocalUse` |
|
|
143
|
+
| 3 | 3.0 latitude/longitude; 3.30 Lambert conformal | `GridDefinition` with `LatLon`, `LambertConformal` |
|
|
144
|
+
| 4 | 4.0 analysis or forecast at a horizontal level or in a layer at a point in time | `ProductDefinition` with `AnalysisForecast` |
|
|
145
|
+
| 5 | 5.0 simple packing; 5.2 complex packing; 5.3 complex packing and spatial differencing | `DataRepresentation` with `SimplePacking`, `ComplexPacking`, `SpatialDifferencing` |
|
|
146
|
+
| 6 | bit map | `Bitmap` |
|
|
147
|
+
| 7 | data, as octets; unpacked by `unpack` for 7.0, 7.2 and 7.3 | `Data` |
|
|
148
|
+
|
|
149
|
+
A section with another template keeps it as `bytes`, so every message
|
|
150
|
+
`parse` accepts decodes and encodes back. `unpack` of a field with another
|
|
151
|
+
data representation template, such as 5.40 (JPEG 2000) or 5.41 (PNG),
|
|
152
|
+
raises `Unsupported`.
|
|
153
|
+
|
|
154
|
+
## Documentation
|
|
155
|
+
|
|
156
|
+
What gnsswire-grib decodes and refuses, what its tests rest on and the
|
|
157
|
+
readings it takes where its specification is ambiguous are under
|
|
158
|
+
[Limitations](https://github.com/tracenav/gnsswire#gnsswire-grib) in the
|
|
159
|
+
[repository README](https://github.com/tracenav/gnsswire#readme), with the
|
|
160
|
+
interface every gnsswire format shares and the version policy. Release
|
|
161
|
+
notes are in [RELEASES.md](https://github.com/tracenav/gnsswire/blob/main/RELEASES.md).
|
|
162
|
+
|
|
163
|
+
## Contribute
|
|
164
|
+
|
|
165
|
+
Report bugs and request templates in the
|
|
166
|
+
[issue tracker](https://github.com/tracenav/gnsswire/issues).
|
|
167
|
+
|
|
168
|
+
## Licence
|
|
169
|
+
|
|
170
|
+
MIT.
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27"] # 1.27: PEP 639 licence metadata
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "gnsswire-grib"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "GRIB edition 2 messages (WMO FM 92): sections, templates and the packed grid point values as integers, read and written losslessly."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.11"
|
|
13
|
+
authors = [{ name = "Trace Navigation" }]
|
|
14
|
+
maintainers = [{ name = "Jacob Armstrong", email = "hi@jacobarm.com" }]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Intended Audience :: Science/Research",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Programming Language :: Python :: 3.13",
|
|
25
|
+
"Programming Language :: Python :: 3.14",
|
|
26
|
+
"Topic :: Scientific/Engineering :: GIS",
|
|
27
|
+
]
|
|
28
|
+
dependencies = ["gnsswire-bits>=0.2.0,<0.3"] # the compatible range of bits (README, Versions)
|
|
29
|
+
|
|
30
|
+
[project.urls]
|
|
31
|
+
Homepage = "https://github.com/tracenav/gnsswire"
|
|
32
|
+
Source = "https://github.com/tracenav/gnsswire/tree/main/packages/grib"
|
|
33
|
+
Changelog = "https://github.com/tracenav/gnsswire/blob/main/RELEASES.md"
|
|
34
|
+
Issues = "https://github.com/tracenav/gnsswire/issues"
|
|
35
|
+
|
|
36
|
+
# gnsswire is a namespace shared by several distributions: no __init__.py at that level.
|
|
37
|
+
[tool.hatch.build.targets.wheel]
|
|
38
|
+
packages = ["src/gnsswire"]
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""GRIB edition 2 messages (WMO FM 92): sections and templates as integers, and the packed values.
|
|
2
|
+
|
|
3
|
+
A message is found in a byte stream with :func:`parse` (one offset) or
|
|
4
|
+
:func:`scan` (every message), the framing of ``_frame``. :func:`decode`
|
|
5
|
+
gives its sections as frozen dataclasses of the integers on the wire, the
|
|
6
|
+
templates of Sections 3, 4 and 5 as field tables where this codec has one
|
|
7
|
+
(``_sections``), and :func:`encode` gives the bytes back:
|
|
8
|
+
``encode(decode(frame)) == frame.data``. Section 7's packed data is kept as
|
|
9
|
+
its octets. :func:`fields` pairs each data section with the sections in
|
|
10
|
+
effect for it (sections 2 to 7 may repeat within a message);
|
|
11
|
+
:func:`unpack` gives the integers X1 + X2 of a field's data points, in the
|
|
12
|
+
order of Section 3, for data representation templates 5.0, 5.2 and 5.3
|
|
13
|
+
(``_unpack``); :func:`scale` applies the decoding formula of Regulation
|
|
14
|
+
92.9.4 to them, Y = (R + X × 2^E) / 10^D, as floats rounded once, apart
|
|
15
|
+
from :func:`decode` and :func:`unpack`, which make no float; :func:`real32`
|
|
16
|
+
gives the reference value R exactly.
|
|
17
|
+
|
|
18
|
+
Source: WMO-No. 306, Manual on Codes, Volume I.2, Part B, FM 92 GRIB
|
|
19
|
+
edition 2: the code form, its notes and Regulations 92.1 to 92.10, the
|
|
20
|
+
"Specifications of octet contents" of Sections 0 to 8, and the templates
|
|
21
|
+
(version 23.0.0, 15 May 2019) cited in each module.
|
|
22
|
+
|
|
23
|
+
Errors: every exception this codec raises for data it cannot decode, unpack
|
|
24
|
+
or encode is an :class:`Error` (a ``ValueError``): :class:`Unsupported`
|
|
25
|
+
(also a ``LookupError``), :class:`FieldError` (also a
|
|
26
|
+
``gnsswire.bits.BitError``), :class:`PackingError` and :class:`NoFrame`
|
|
27
|
+
(also a ``TypeError``).
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from ._frame import Error, Frame, Invalid, NeedMore, parse, scan
|
|
31
|
+
from ._message import Field, FieldError, Message, NoFrame, Unsupported, decode, encode, fields
|
|
32
|
+
from ._sections import (
|
|
33
|
+
GRID_TEMPLATES,
|
|
34
|
+
PRODUCT_TEMPLATES,
|
|
35
|
+
REPRESENTATION_TEMPLATES,
|
|
36
|
+
AnalysisForecast,
|
|
37
|
+
Bitmap,
|
|
38
|
+
ComplexPacking,
|
|
39
|
+
Data,
|
|
40
|
+
DataRepresentation,
|
|
41
|
+
GridDefinition,
|
|
42
|
+
Identification,
|
|
43
|
+
LambertConformal,
|
|
44
|
+
LatLon,
|
|
45
|
+
LocalUse,
|
|
46
|
+
ProductDefinition,
|
|
47
|
+
Section,
|
|
48
|
+
SimplePacking,
|
|
49
|
+
SpatialDifferencing,
|
|
50
|
+
)
|
|
51
|
+
from ._unpack import (
|
|
52
|
+
ABSENT,
|
|
53
|
+
MISSING_PRIMARY,
|
|
54
|
+
MISSING_SECONDARY,
|
|
55
|
+
PRESENT,
|
|
56
|
+
PackingError,
|
|
57
|
+
Unpacked,
|
|
58
|
+
real32,
|
|
59
|
+
scale,
|
|
60
|
+
unpack,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
__all__ = [
|
|
64
|
+
"ABSENT", "GRID_TEMPLATES", "MISSING_PRIMARY", "MISSING_SECONDARY", "PRESENT",
|
|
65
|
+
"PRODUCT_TEMPLATES", "REPRESENTATION_TEMPLATES", "AnalysisForecast", "Bitmap",
|
|
66
|
+
"ComplexPacking", "Data", "DataRepresentation", "Error", "Field", "FieldError", "Frame",
|
|
67
|
+
"GridDefinition", "Identification", "Invalid", "LambertConformal", "LatLon", "LocalUse",
|
|
68
|
+
"Message", "NeedMore", "NoFrame", "PackingError", "ProductDefinition", "Section",
|
|
69
|
+
"SimplePacking", "SpatialDifferencing", "Unpacked", "Unsupported", "decode", "encode",
|
|
70
|
+
"fields", "parse", "real32", "scale", "scan", "unpack",
|
|
71
|
+
]
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""Framing: :func:`parse` and :func:`scan`.
|
|
2
|
+
|
|
3
|
+
Source: WMO-No. 306, Manual on Codes, Volume I.2, Part B, FM 92 GRIB
|
|
4
|
+
(edition 2), its notes and Regulations 92.1 to 92.10, and "Specifications
|
|
5
|
+
of octet contents", Sections 0 to 8.
|
|
6
|
+
|
|
7
|
+
A message is:
|
|
8
|
+
|
|
9
|
+
=========== ========== =================================================
|
|
10
|
+
section octets content
|
|
11
|
+
=========== ========== =================================================
|
|
12
|
+
0 16 "GRIB", 2 reserved octets, discipline (Code table
|
|
13
|
+
0.0), edition (2), total length of the message (8
|
|
14
|
+
octets, Section 0 included; Regulation 92.2)
|
|
15
|
+
1 to 7 each its octets 1-4 the section's length, octet 5 its
|
|
16
|
+
own length number (Regulation 92.3.1, 92.3.2 and the
|
|
17
|
+
regulations that apply them to Sections 2 to 7)
|
|
18
|
+
8 4 "7777" (Regulation 92.10.1)
|
|
19
|
+
=========== ========== =================================================
|
|
20
|
+
|
|
21
|
+
Note 4 of the code form: "Sequences of GRIB sections 2 to 7, sections 3 to 7
|
|
22
|
+
or sections 4 to 7 may be repeated within a single GRIB message. All
|
|
23
|
+
sections within such repeated sequences must be present and shall appear in
|
|
24
|
+
the numerical order noted above." Section 2 is optional (92.4.2). So the
|
|
25
|
+
sections after Section 0 are 1, then 2 or 3; 2 is followed by 3, 3 by 4 and
|
|
26
|
+
so on to 7; after 7 comes 2, 3, 4 or the end. :func:`parse` checks this order
|
|
27
|
+
and that the sections fill the message exactly, as a CRC would be checked in
|
|
28
|
+
a format that had one: GRIB has no checksum, and the walk of section lengths
|
|
29
|
+
from "GRIB" to "7777" is what tells a message from bytes that start with
|
|
30
|
+
"GRIB" by chance.
|
|
31
|
+
|
|
32
|
+
A message of another edition (octet 8; GRIB edition 1 has its edition number
|
|
33
|
+
in the same octet) is :class:`Invalid` here: it is not a frame of this codec.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
from collections.abc import Iterator
|
|
39
|
+
from dataclasses import dataclass
|
|
40
|
+
from typing import TYPE_CHECKING
|
|
41
|
+
|
|
42
|
+
if TYPE_CHECKING:
|
|
43
|
+
from collections.abc import Buffer # Python 3.12; annotations only
|
|
44
|
+
|
|
45
|
+
START = b"GRIB"
|
|
46
|
+
END = b"7777"
|
|
47
|
+
EDITION = 2
|
|
48
|
+
INDICATOR = 16 # Section 0 (Regulation 92.2.1)
|
|
49
|
+
END_LENGTH = 4 # Section 8 (Regulation 92.10.1)
|
|
50
|
+
|
|
51
|
+
# The sections that may follow each section (note 4 of the code form); 8 is the end.
|
|
52
|
+
NEXT: dict[int, frozenset[int]] = {
|
|
53
|
+
0: frozenset({1}), 1: frozenset({2, 3}), 2: frozenset({3}), 3: frozenset({4}),
|
|
54
|
+
4: frozenset({5}), 5: frozenset({6}), 6: frozenset({7}), 7: frozenset({2, 3, 4, 8}),
|
|
55
|
+
}
|
|
56
|
+
# The fewest octets each section can have: the octets its specification
|
|
57
|
+
# lists before any template or variable part.
|
|
58
|
+
MIN_LENGTH = {1: 21, 2: 5, 3: 14, 4: 9, 5: 11, 6: 6, 7: 5}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class Error(ValueError):
|
|
62
|
+
"""Base of every exception this codec raises for data, frames or messages."""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True, slots=True)
|
|
66
|
+
class Frame:
|
|
67
|
+
"""One GRIB edition 2 message whose sections fill it from "GRIB" to "7777"."""
|
|
68
|
+
|
|
69
|
+
offset: int # where the message starts in the buffer that was parsed
|
|
70
|
+
length: int # octets of the message (Section 0's total length)
|
|
71
|
+
data: bytes # the whole message, Section 0 to Section 8
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass(frozen=True, slots=True)
|
|
75
|
+
class NeedMore:
|
|
76
|
+
"""The bytes at the offset could start a message, but it is not all here yet."""
|
|
77
|
+
|
|
78
|
+
needed: int # total bytes required from the offset to decide
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@dataclass(frozen=True, slots=True)
|
|
82
|
+
class Invalid:
|
|
83
|
+
"""No valid message starts at the offset."""
|
|
84
|
+
|
|
85
|
+
reason: str
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _check_offset(buf: Buffer, offset: int) -> None:
|
|
89
|
+
if not 0 <= offset <= len(buf): # type: ignore[arg-type]
|
|
90
|
+
raise Error(f"offset {offset} outside buffer of {len(buf)} bytes") # type: ignore[arg-type]
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def sections(data: bytes | bytearray | memoryview, start: int, end: int) -> list[tuple[int, int, int]]:
|
|
94
|
+
"""``(number, offset, length)`` of each section from ``start`` to ``end``.
|
|
95
|
+
|
|
96
|
+
``end`` is where Section 8 starts. Raises ``ValueError`` (not this
|
|
97
|
+
codec's :class:`Error`: callers turn it into :class:`Invalid` or their
|
|
98
|
+
own error) naming what breaks the order of note 4.
|
|
99
|
+
"""
|
|
100
|
+
found = []
|
|
101
|
+
pos, prev = start, 0
|
|
102
|
+
while pos < end:
|
|
103
|
+
if end - pos < 5:
|
|
104
|
+
raise ValueError(f"{end - pos} octets before Section 8 are not a section")
|
|
105
|
+
length = int.from_bytes(data[pos:pos + 4], "big")
|
|
106
|
+
number = data[pos + 4]
|
|
107
|
+
if number not in NEXT[prev]:
|
|
108
|
+
raise ValueError(f"section {number} after section {prev}")
|
|
109
|
+
if length < MIN_LENGTH[number]:
|
|
110
|
+
raise ValueError(f"section {number} of {length} octets is shorter than "
|
|
111
|
+
f"{MIN_LENGTH[number]}")
|
|
112
|
+
if pos + length > end:
|
|
113
|
+
raise ValueError(f"section {number} of {length} octets runs past Section 8")
|
|
114
|
+
found.append((number, pos, length))
|
|
115
|
+
pos += length
|
|
116
|
+
prev = number
|
|
117
|
+
if 8 not in NEXT[prev]:
|
|
118
|
+
raise ValueError(f"the message ends after section {prev}, not section 7")
|
|
119
|
+
return found
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def parse(buf: Buffer, offset: int = 0) -> Frame | NeedMore | Invalid:
|
|
123
|
+
"""Is there a valid GRIB edition 2 message starting exactly at ``offset``? Never searches.
|
|
124
|
+
|
|
125
|
+
``buf`` is any bytes-like object (``bytes``, ``bytearray``, ``memoryview``...).
|
|
126
|
+
"""
|
|
127
|
+
view = buf if isinstance(buf, (bytes, bytearray)) else memoryview(buf).cast("B")
|
|
128
|
+
_check_offset(view, offset)
|
|
129
|
+
have = len(view) - offset
|
|
130
|
+
head = bytes(view[offset:offset + 4])
|
|
131
|
+
if head != START[:len(head)]:
|
|
132
|
+
return Invalid('no "GRIB"')
|
|
133
|
+
if have < INDICATOR:
|
|
134
|
+
return NeedMore(INDICATOR)
|
|
135
|
+
edition = view[offset + 7]
|
|
136
|
+
if edition != EDITION:
|
|
137
|
+
return Invalid(f"GRIB edition {edition}")
|
|
138
|
+
total = int.from_bytes(view[offset + 8:offset + 16], "big")
|
|
139
|
+
if total < INDICATOR + MIN_LENGTH[1] + END_LENGTH:
|
|
140
|
+
return Invalid(f"total length {total} is too short for a message")
|
|
141
|
+
if have < total:
|
|
142
|
+
return NeedMore(total)
|
|
143
|
+
end = offset + total - END_LENGTH
|
|
144
|
+
if bytes(view[end:end + 4]) != END:
|
|
145
|
+
return Invalid('no "7777" where the total length ends')
|
|
146
|
+
try:
|
|
147
|
+
sections(view, offset + INDICATOR, end)
|
|
148
|
+
except ValueError as exc:
|
|
149
|
+
return Invalid(str(exc))
|
|
150
|
+
return Frame(offset, total, bytes(view[offset:offset + total]))
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def scan(buf: Buffer, offset: int = 0) -> Iterator[Frame]:
|
|
154
|
+
"""Every valid message from ``offset`` on, skipping one byte after anything else.
|
|
155
|
+
|
|
156
|
+
The buffer is taken as complete: a message cut off at its end is skipped
|
|
157
|
+
like any other invalid bytes. Only positions holding "GRIB" are tried.
|
|
158
|
+
"""
|
|
159
|
+
data = buf if isinstance(buf, (bytes, bytearray)) else bytes(memoryview(buf))
|
|
160
|
+
_check_offset(data, offset)
|
|
161
|
+
pos = offset
|
|
162
|
+
while (pos := data.find(START, pos)) >= 0:
|
|
163
|
+
result = parse(data, pos)
|
|
164
|
+
if isinstance(result, Frame):
|
|
165
|
+
yield result
|
|
166
|
+
pos += result.length
|
|
167
|
+
else:
|
|
168
|
+
pos += 1
|