simple-dwd-weatherforecast 2.1.1__py3-none-any.whl → 2.1.2__py3-none-any.whl

simple_dwd_weatherforecast/dwdforecast.py

@@ -7,6 +7,8 @@ from datetime import datetime, timedelta, timezone
 from enum import Enum
 from io import BytesIO
 from zipfile import ZipFile
+import httpx
+from stream_unzip import stream_unzip
 
 import arrow
 import requests
@@ -677,7 +679,7 @@ class Weather:
 
         items = []
         result = kmlTree.xpath(
-            './kml:ExtendedData/dwd:Forecast[@dwd:elementName="{}"]/dwd:value'.format(
+            './kml:Document/kml:Placemark/kml:ExtendedData/dwd:Forecast[@dwd:elementName="{}"]/dwd:value'.format(
                 weatherDataType.value[0]
             ),
             namespaces=self.namespaces,
@@ -694,14 +696,11 @@ class Weather:
                 items.append(None)
         return items
 
-    def parse_kml(self, kml, force_hourly=False):
-        stream = etree.iterparse(kml)
-        (_, tree) = next(stream)
+    def parse_kml(self, kml):
+        tree = etree.fromstring(kml)  # type: ignore
         timesteps = self.parse_timesteps(tree)
         issue_time_new = self.parse_issue_time(tree)
-        tree.clear()
 
-        tree = self.parse_placemark(stream)
         self.issue_time = issue_time_new
 
         self.loaded_station_name = self.parse_station_name(tree)
@@ -748,17 +747,6 @@ class Weather:
             for (i, t) in enumerate(timesteps)
         )
 
-    def parse_placemark(self, stream):
-        for _, tree in stream:
-            for placemark in tree.findall(
-                ".//kml:Placemark", namespaces=self.namespaces
-            ):
-                item = placemark.find(".//kml:name", namespaces=self.namespaces)
-
-                if item is not None and item.text == self.station_id:
-                    return placemark
-                # placemark.clear()
-
     def parse_issue_time(self, tree):
         issue_time_new = arrow.get(
             tree.xpath("//dwd:IssueTime", namespaces=self.namespaces)[0].text,
@@ -767,7 +755,9 @@ class Weather:
         return issue_time_new
 
     def parse_station_name(self, tree):
-        return tree.xpath("./kml:description", namespaces=self.namespaces)[0].text
+        return tree.xpath(
+            "./kml:Document/kml:Placemark/kml:description", namespaces=self.namespaces
+        )[0].text
 
     def parse_timesteps(self, tree):
         return [
@@ -781,7 +771,7 @@ class Weather:
         return [
             elem.split(".")[0]
             for elem in tree.xpath(
-                './kml:ExtendedData/dwd:Forecast[@dwd:elementName="ww"]/dwd:value',
+                './kml:Document/kml:Placemark/kml:ExtendedData/dwd:Forecast[@dwd:elementName="ww"]/dwd:value',
                 namespaces=self.namespaces,
             )[0].text.split()
         ]
@@ -945,11 +935,8 @@ class Weather:
         except Exception as error:
             print(f"Error in download_weather_report: {type(error)} args: {error.args}")
 
-    def download_latest_kml(self, stationid, force_hourly=False):
-        if force_hourly:
-            url = "https://opendata.dwd.de/weather/local_forecasts/mos/MOSMIX_S/all_stations/kml/MOSMIX_S_LATEST_240.kmz"
-        else:
-            url = f"https://opendata.dwd.de/weather/local_forecasts/mos/MOSMIX_L/single_stations/{stationid}/kml/MOSMIX_L_LATEST_{stationid}.kmz"
+    def download_small_kml(self, stationid) -> bytes | None:
+        url = f"https://opendata.dwd.de/weather/local_forecasts/mos/MOSMIX_L/single_stations/{stationid}/kml/MOSMIX_L_LATEST_{stationid}.kmz"
         headers = {"If-None-Match": self.etags[url] if url in self.etags else ""}  # type: ignore
         try:
             request = requests.get(url, headers=headers, timeout=30)
@@ -958,12 +945,73 @@ class Weather:
                 return
             self.etags[url] = request.headers["ETag"]  # type: ignore
             with ZipFile(BytesIO(request.content), "r") as kmz:
-                # large RAM allocation
                 with kmz.open(kmz.namelist()[0], "r") as kml:
-                    self.parse_kml(kml, force_hourly)
+                    return kml.read()
+
         except Exception as error:
             print(f"Error in download_latest_kml: {type(error)} args: {error.args}")
 
+    def get_chunks(self):
+        def zipped_chunks():
+            # Iterable that yields the bytes of a zip file
+            with httpx.stream(
+                "GET",
+                "https://opendata.dwd.de/weather/local_forecasts/mos/MOSMIX_S/all_stations/kml/MOSMIX_S_LATEST_240.kmz",
+            ) as r:
+                yield from r.iter_bytes(chunk_size=131072)
+
+        return stream_unzip(zipped_chunks())
+
+    def download_large_kml(self, stationid):
+        placemark = b""
+
+        for file_name, file_size, unzipped_chunks in self.get_chunks():
+            chunk1 = b""
+            chunk2 = b""
+            first_chunk = None
+
+            save_next = False
+            save_next_next = False
+            stop = False
+            # unzipped_chunks must be iterated to completion or UnfinishedIterationError will be raised
+            for chunk in unzipped_chunks:
+                if stop:
+                    continue
+                if not first_chunk:
+                    first_chunk = chunk
+                if save_next_next:
+                    placemark = chunk1 + chunk2 + chunk
+                    save_next_next = False
+                    stop = True
+                if save_next:
+                    chunk2 = chunk
+                    save_next_next = True
+                    save_next = False
+
+                if stationid.encode() in chunk:
+                    chunk1 = chunk
+                    save_next = True
+            if first_chunk:
+                start = placemark.find(b"<kml:Placemark>\n")
+
+                result = (
+                    first_chunk[: first_chunk.find(b"<kml:Placemark>")]
+                    + placemark[
+                        start : placemark.find(b"</kml:Placemark>\n", start) + 17
+                    ]
+                    + b"</kml:Document></kml:kml>"
+                )
+                return result
+
+    def download_latest_kml(self, stationid, force_hourly=False):
+        kml = (
+            self.download_large_kml(stationid)
+            if force_hourly
+            else self.download_small_kml(stationid)
+        )
+        if kml is not None:
+            self.parse_kml(kml)
+
     def download_latest_report(self):
         station_id = self.station_id
         if len(station_id) == 4:

simple_dwd_weatherforecast/dwdmap.py

@@ -95,7 +95,15 @@ def get_map(
         raise ValueError(
             "Width and height must not exceed 1200 and 1400 respectively. Please be kind to the DWD servers."
         )
-    url = f"https://maps.dwd.de/geoserver/dwd/wms?service=WMS&version=1.1.0&request=GetMap&layers={map_type.value},{background_type.value}&bbox={minx},{miny},{maxx},{maxy}&width={image_width}&height={image_height}&srs=EPSG:4326&styles=&format=image/png"
+    if background_type in [
+        WeatherBackgroundMapType.SATELLIT,
+        WeatherBackgroundMapType.KREISE,
+        WeatherBackgroundMapType.GEMEINDEN,
+    ]:
+        layers = f"{background_type.value}, {map_type.value}"
+    else:
+        layers = f"{map_type.value}, {background_type.value}"
+    url = f"https://maps.dwd.de/geoserver/dwd/wms?service=WMS&version=1.1.0&request=GetMap&layers={layers}&bbox={minx},{miny},{maxx},{maxy}&width={image_width}&height={image_height}&srs=EPSG:4326&styles=&format=image/png"
     request = requests.get(url, stream=True)
     if request.status_code == 200:
         image = Image.open(BytesIO(request.content))
@@ -135,6 +143,8 @@ class ImageLoop:
         self._miny = miny
         self._maxx = maxx
         self._maxy = maxy
+        if map_type != WeatherMapType.NIEDERSCHLAGSRADAR:
+            raise ValueError("Only NIEDERSCHLAGSRADAR is supported in a loop")
         self._map_type = map_type
         self._background_type = background_type
         self._steps = steps
@@ -173,7 +183,15 @@ class ImageLoop:
                 self._images.append(self._get_image(self._last_update))
 
     def _get_image(self, date: datetime) -> ImageFile.ImageFile:
-        url = f"https://maps.dwd.de/geoserver/dwd/wms?service=WMS&version=1.1.0&request=GetMap&layers={self._map_type.value},{self._background_type.value}&bbox={self._minx},{self._miny},{self._maxx},{self._maxy}&width={self._image_width}&height={self._image_height}&srs=EPSG:4326&styles=&format=image/png&TIME={date.strftime("%Y-%m-%dT%H:%M:00.0Z")}"
+        if self._background_type in [
+            WeatherBackgroundMapType.SATELLIT,
+            WeatherBackgroundMapType.KREISE,
+            WeatherBackgroundMapType.GEMEINDEN,
+        ]:
+            layers = f"{self._background_type.value}, {self._map_type.value}"
+        else:
+            layers = f"{self._map_type.value}, {self._background_type.value}"
+        url = f"https://maps.dwd.de/geoserver/dwd/wms?service=WMS&version=1.1.0&request=GetMap&layers={layers}&bbox={self._minx},{self._miny},{self._maxx},{self._maxy}&width={self._image_width}&height={self._image_height}&srs=EPSG:4326&styles=&format=image/png&TIME={date.strftime("%Y-%m-%dT%H:%M:00.0Z")}"
         request = requests.get(url, stream=True)
         if request.status_code != 200:
             raise ConnectionError("Error during image request from DWD servers")

{simple_dwd_weatherforecast-2.1.1.dist-info → simple_dwd_weatherforecast-2.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: simple_dwd_weatherforecast
-Version: 2.1.1
+Version: 2.1.2
 Summary: A simple tool to retrieve a weather forecast from DWD OpenData
 Home-page: https://github.com/FL550/simple_dwd_weatherforecast.git
 Author: Max Fermor
@@ -21,6 +21,8 @@ Requires-Dist: arrow
 
 DISCLAIMER: This project is a private open source project and doesn't have any connection with Deutscher Wetterdienst.
 
+**If you like my work, I would be really happy if you buy me some coffee: [Buy Me A Coffee][buymecoffee]**
+
 ## Weather data
 
 This is a python package for simple access to hourly forecast data for the next 10 days. The data is updated every six hours and updated when needed. Some stations also have actual reported weather, which you can also retrieve.
@@ -208,7 +210,7 @@ get_map(minx,miny,maxx,maxy, map_type: WeatherMapType, background_type: WeatherB
 
 ### Image loop
 
-There is also the possibility to retrieve multiple images as a loop. This can be done by the class ImageLoop.
+There is also the possibility to retrieve multiple images as a loop. This can be done by the class ImageLoop. This however only works for the precipitation radar.
 
 
 #### Usage example

{simple_dwd_weatherforecast-2.1.1.dist-info → simple_dwd_weatherforecast-2.1.2.dist-info}/RECORD

@@ -1,6 +1,6 @@
 simple_dwd_weatherforecast/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-simple_dwd_weatherforecast/dwdforecast.py,sha256=ctnQAFJS8UID9KZCXK7O5FyBTlgkqvoNLVJG5E3TkCY,36694
-simple_dwd_weatherforecast/dwdmap.py,sha256=xIgUiyDd6gHNDGlDL2hp-Koz11s1XPtMdeVwj_1Pmnk,6020
+simple_dwd_weatherforecast/dwdforecast.py,sha256=te_MIe6GzZ32UWrDTpbqJdHk0hFhs9dqN4h2rS3J-yA,38247
+simple_dwd_weatherforecast/dwdmap.py,sha256=cPCcL1u5qeIEDLBcL0qOH0-7dPIi3mge4-PUqfZlRQc,6737
 simple_dwd_weatherforecast/stations.json,sha256=1u8qc2CT_rVy49SAlOicGixzHln6Y0FXevuFAz2maBw,838948
 simple_dwd_weatherforecast/uv_stations.json,sha256=ADenYo-aR6qbf0UFkfYr72kkFzL9HyUKe4VQ23POGF8,2292
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -26,17 +26,17 @@ tests/test_is_in_timerange.py,sha256=3y88L3N73NxSTJ-_edx6OCnxHWKJWWFma98gjZvJDGg
 tests/test_is_valid_timeframe.py,sha256=mXjeu3lUyixiBUEljirTf6qDM_FZFQGWa-Rk0NBMUDU,891
 tests/test_location_tools.py,sha256=wto_XzVnARJQ-Qc83YAn0ahfMBSaOHpfzqAeKRDsNm8,1208
 tests/test_map.py,sha256=uKxNjMXLFT3pczZKLqkfPK5xaVfmql-r5L9VPgCbS3Q,5671
-tests/test_parsekml.py,sha256=Y0EHJi5e8lqFUS8N9yfA5ByQkKV4-iyrwUaIipQpM7w,1065
+tests/test_parsekml.py,sha256=mpje7FoMIz566BLW-Fr_69cxLyS7w4qvrSOIlxVYoUU,1625
 tests/test_region.py,sha256=ReUB9Cy9roBemkpEkTjZZav-Mu3Ha7ADOAfa9J-gi80,877
 tests/test_reported_weather.py,sha256=ULg4ogZRxus01p2rdxiSFL75AisqtcvnLDOc7uJMBH0,767
 tests/test_station.py,sha256=Zjx-q0yxKVxVI_L1yB_bqY5pjZPoa1L94uC8Gx6shdY,1026
 tests/test_stationsfile.py,sha256=slRH5N4Gznr6tkN2oMFWJbVCw3Xrma7Hvzn1lG5E-Qg,1401
-tests/test_update.py,sha256=r763R-MfqFqQQy43PcmE9yWAiQU6T2rQ1u55ujlwMt8,7216
-tests/test_update_hourly.py,sha256=mUc66JHndgIMdZ0DD_KXQTbPPg1g86F6CPUwJYjBV5U,1619
+tests/test_update.py,sha256=AIzzHMxcjwQjeTB0l3YFgB7HkGDbuqiHofwy41mS0m4,7440
+tests/test_update_hourly.py,sha256=7Zl8ml3FTdqw3_Qwr_Tz-sWTzypvrBWmxeig2Vwp_ZQ,1781
 tests/test_uv_index.py,sha256=tr6wnOyHlXT1S3yp1oeHc4-Brmc-EMEdM4mtyrdpcHg,579
 tests/test_weather.py,sha256=ZyX4ldUoJpJp7YpiNQwU6Od-nYRay-3qcaDJdNq8fhY,780
-simple_dwd_weatherforecast-2.1.1.dist-info/LICENCE,sha256=27UG7gteqvSWuZlsbIq2_OAbh7VyifGGl-1zpuUoBcw,1072
-simple_dwd_weatherforecast-2.1.1.dist-info/METADATA,sha256=fu3NJyDqNgLScF7xwu9sTBrJNYEZ0UpmpSGxipZdlqk,11893
-simple_dwd_weatherforecast-2.1.1.dist-info/WHEEL,sha256=uCRv0ZEik_232NlR4YDw4Pv3Ajt5bKvMH13NUU7hFuI,91
-simple_dwd_weatherforecast-2.1.1.dist-info/top_level.txt,sha256=iyEobUh14Tzitx39Oi8qm0NhBrnZovl_dNKtvLUkLEM,33
-simple_dwd_weatherforecast-2.1.1.dist-info/RECORD,,
+simple_dwd_weatherforecast-2.1.2.dist-info/LICENCE,sha256=27UG7gteqvSWuZlsbIq2_OAbh7VyifGGl-1zpuUoBcw,1072
+simple_dwd_weatherforecast-2.1.2.dist-info/METADATA,sha256=U8Nf45jVUBJipMq-HRG7zd9ILGyn_ezc_dv6ctF9uGs,12054
+simple_dwd_weatherforecast-2.1.2.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
+simple_dwd_weatherforecast-2.1.2.dist-info/top_level.txt,sha256=iyEobUh14Tzitx39Oi8qm0NhBrnZovl_dNKtvLUkLEM,33
+simple_dwd_weatherforecast-2.1.2.dist-info/RECORD,,

{simple_dwd_weatherforecast-2.1.1.dist-info → simple_dwd_weatherforecast-2.1.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (74.1.1)
+Generator: setuptools (74.1.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

tests/test_parsekml.py

@@ -1,5 +1,6 @@
 from datetime import datetime, timezone
 import unittest
+from unittest.mock import patch
 
 from simple_dwd_weatherforecast import dwdforecast
 import dummy_data
@@ -14,9 +15,24 @@ class KMLParseTestCase(unittest.TestCase):
 
     def test_parse_kml(self):
         with open(self.FILE_NAME, "rb") as kml:
-            self.dwd_weather.parse_kml(kml)
+            self.dwd_weather.parse_kml(kml.read())
             self.assertEqual(self.dwd_weather.forecast_data, dummy_data.parsed_data)
-            self.assertEqual(self.dwd_weather.issue_time, datetime(2020, 11, 6, 3, 0, tzinfo=timezone.utc))
+            self.assertEqual(
+                self.dwd_weather.issue_time,
+                datetime(2020, 11, 6, 3, 0, tzinfo=timezone.utc),
+            )
+
+
+def helper():
+    result = []
+    read_size = 131072
+    # Iterable that yields the bytes of a zip file
+    with open("development/MOSMIX_L_2023100809_stripped.kml", "rb") as kml:
+        content = kml.read(read_size)
+        while len(content) > 0:
+            result.append(content)
+            content = kml.read(read_size)
+    return zip([0], [0], [result])
 
 
 class KMLParseFullTestCase(unittest.TestCase):
@@ -25,9 +41,12 @@ class KMLParseFullTestCase(unittest.TestCase):
     def setUp(self):
         self.dwd_weather = dwdforecast.Weather("L511")
 
-    def test_parse_kml(self):
-        with open(self.FILE_NAME, "rb") as kml:
-            self.dwd_weather.parse_kml(kml)
-            self.assertEqual(
-                self.dwd_weather.forecast_data, dummy_data_full.parsed_data
-            )
+    @patch(
+        "simple_dwd_weatherforecast.dwdforecast.Weather.get_chunks",
+        return_value=helper(),
+    )
+    def test_parse_kml(self, _):
+        self.dwd_weather.download_latest_kml(
+            self.dwd_weather.station_id, force_hourly=True
+        )
+        self.assertEqual(self.dwd_weather.forecast_data, dummy_data_full.parsed_data)

tests/test_update.py

@@ -22,6 +22,9 @@ class WeatherUpdate(unittest.TestCase):
     def test_download(self, _1, _2):
         self.dwd_weather.update()
         self.assertIsNotNone(self.dwd_weather.forecast_data)
+        self.assertIsNotNone(self.dwd_weather.forecast_data)
+        self.assertEqual(self.dwd_weather.station_id, "H889")
+        self.assertEqual(self.dwd_weather.issue_time.date(), datetime.now().date())  # type: ignore
 
     @patch(
         "simple_dwd_weatherforecast.dwdforecast.Weather.download_latest_report",
@@ -55,7 +58,7 @@ class WeatherUpdate(unittest.TestCase):
         self.dwd_weather.issue_time = datetime(
             *(time.strptime("2020-11-06T03:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ")[0:6]),
             0,
-            timezone.utc
+            timezone.utc,
         )
         self.dwd_weather.update()
         mock_function.assert_called()

tests/test_update_hourly.py

@@ -14,6 +14,8 @@ class WeatherUpdate(unittest.TestCase):
     def test_download(self):
         self.dwd_weather.update(force_hourly=True)
         self.assertIsNotNone(self.dwd_weather.forecast_data)
+        self.assertEqual(self.dwd_weather.station_id, "H889")
+        self.assertEqual(self.dwd_weather.issue_time.date(), datetime.now().date())  # type: ignore
 
     @patch(
         "simple_dwd_weatherforecast.dwdforecast.Weather.download_latest_kml",