isd-fetch 0.1.0__tar.gz

isd_fetch-0.1.0/LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2024, Cyril
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

isd_fetch-0.1.0/PKG-INFO

@@ -0,0 +1,99 @@
+Metadata-Version: 2.1
+Name: isd-fetch
+Version: 0.1.0
+Summary: A Python package for manipulating and analyzing ISD data.
+Author: Cyril Joly
+License: BSD 3-Clause License
+        
+        Copyright (c) 2024, Cyril
+        
+        Redistribution and use in source and binary forms, with or without
+        modification, are permitted provided that the following conditions are met:
+        
+        1. Redistributions of source code must retain the above copyright notice, this
+           list of conditions and the following disclaimer.
+        
+        2. Redistributions in binary form must reproduce the above copyright notice,
+           this list of conditions and the following disclaimer in the documentation
+           and/or other materials provided with the distribution.
+        
+        3. Neither the name of the copyright holder nor the names of its
+           contributors may be used to endorse or promote products derived from
+           this software without specific prior written permission.
+        
+        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+        AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+        IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+        DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+        FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+        DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+        SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+        CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+        OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+        OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+        
+Keywords: isd,noaa
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: geopandas
+Requires-Dist: tqdm
+
+# PyISD: A Python Package for NOAA's ISD Lite Dataset
+
+**PyISD** is a Python package designed for loading and processing NOAA's ISD Lite dataset. The dataset, as described by NOAA, is a streamlined version of the full Integrated Surface Database (ISD). It includes eight common surface parameters in a fixed-width format, free of duplicate values, sub-hourly data, and complicated flags, making it suitable for general research and scientific purposes. For more details, visit the [official ISD homepage](https://www.ncei.noaa.gov/products/land-based-station/integrated-surface-database).
+
+## **Features**
+- Load and query the ISD Lite dataset with ease.
+- Retrieve and process metadata for stations worldwide.
+- Filter data based on spatial and temporal constraints.
+
+## **Example Usage**
+
+### **1. Importing and Loading Metadata**
+You can start by importing the `IsdLite` module, fetching metadata for weather stations worldwide and displaying a sample of the station metadata:
+
+```python
+from pyisd import IsdLite
+
+CRS = 4326
+
+module = IsdLite(crs=CRS, verbose=True)
+module.raw_metadata.sample(5)
+```
+
+The output displays station metadata including station name, latitude, longitude, elevation, and the period of available records:
+
+```
+         USAF   WBAN             STATION NAME CTRY   ST CALL     LAT      LON  \
+8480   377350  99999                   GANDJA   AJ  NaN  NaN  40.717   46.417   
+1023   027710  99999  JOUTSA LEIVONMAKI SAVEN   FI  NaN  NaN  61.883   26.100   
+11880  545340  99999                 TANGSHAN   CH  NaN  NaN  39.650  118.100   
+3795   111900  99999               EISENSTADT   AU  NaN  NaN  47.850   16.533   
+26693  957119  99999     WEST WYALONG AIRPORT   AS  NaN  NaN -33.930  147.200   
+
+        ELEV(M)     BEGIN       END        x       y               geometry  
+8480     309.0  19320101  20241117   46.417  40.717  POINT (46.417 40.717)  
+1023     146.0  20080115  20241112   26.100  61.883    POINT (26.1 61.883)  
+11880     29.0  19560820  20241112  118.100  39.650    POINT (118.1 39.65)  
+3795     189.3  19730627  20241117   16.533  47.850   POINT (16.533 47.85)  
+26693    262.0  19651231  19840629  147.200 -33.930   POINT (147.2 -33.93)  
+```
+
+### **2. Fetching and Visualizing Data**
+To retrieve data, you can specify the time period and spatial constraints. Here, we fetch temperature data (`temp`) for the bounding box around Paris between January 1, 2023, and November 20, 2024:
+
+```python
+from pyisd.misc import get_box
+
+geometry = get_box(place='Paris', width=1., crs=CRS)
+
+data = module.get_data(start=20230101, end=20241120, geometry=geometry, organize_by='field')
+
+data['temp'].plot(figsize=(10, 4), legend=False, c='grey', lw=0.6)
+```
+
+![time_series](https://github.com/CyrilJl/pyisd/blob/main/assets/temp_time_series.png?raw=true)

isd_fetch-0.1.0/README.md

@@ -0,0 +1,55 @@
+# PyISD: A Python Package for NOAA's ISD Lite Dataset
+
+**PyISD** is a Python package designed for loading and processing NOAA's ISD Lite dataset. The dataset, as described by NOAA, is a streamlined version of the full Integrated Surface Database (ISD). It includes eight common surface parameters in a fixed-width format, free of duplicate values, sub-hourly data, and complicated flags, making it suitable for general research and scientific purposes. For more details, visit the [official ISD homepage](https://www.ncei.noaa.gov/products/land-based-station/integrated-surface-database).
+
+## **Features**
+- Load and query the ISD Lite dataset with ease.
+- Retrieve and process metadata for stations worldwide.
+- Filter data based on spatial and temporal constraints.
+
+## **Example Usage**
+
+### **1. Importing and Loading Metadata**
+You can start by importing the `IsdLite` module, fetching metadata for weather stations worldwide and displaying a sample of the station metadata:
+
+```python
+from pyisd import IsdLite
+
+CRS = 4326
+
+module = IsdLite(crs=CRS, verbose=True)
+module.raw_metadata.sample(5)
+```
+
+The output displays station metadata including station name, latitude, longitude, elevation, and the period of available records:
+
+```
+         USAF   WBAN             STATION NAME CTRY   ST CALL     LAT      LON  \
+8480   377350  99999                   GANDJA   AJ  NaN  NaN  40.717   46.417   
+1023   027710  99999  JOUTSA LEIVONMAKI SAVEN   FI  NaN  NaN  61.883   26.100   
+11880  545340  99999                 TANGSHAN   CH  NaN  NaN  39.650  118.100   
+3795   111900  99999               EISENSTADT   AU  NaN  NaN  47.850   16.533   
+26693  957119  99999     WEST WYALONG AIRPORT   AS  NaN  NaN -33.930  147.200   
+
+        ELEV(M)     BEGIN       END        x       y               geometry  
+8480     309.0  19320101  20241117   46.417  40.717  POINT (46.417 40.717)  
+1023     146.0  20080115  20241112   26.100  61.883    POINT (26.1 61.883)  
+11880     29.0  19560820  20241112  118.100  39.650    POINT (118.1 39.65)  
+3795     189.3  19730627  20241117   16.533  47.850   POINT (16.533 47.85)  
+26693    262.0  19651231  19840629  147.200 -33.930   POINT (147.2 -33.93)  
+```
+
+### **2. Fetching and Visualizing Data**
+To retrieve data, you can specify the time period and spatial constraints. Here, we fetch temperature data (`temp`) for the bounding box around Paris between January 1, 2023, and November 20, 2024:
+
+```python
+from pyisd.misc import get_box
+
+geometry = get_box(place='Paris', width=1., crs=CRS)
+
+data = module.get_data(start=20230101, end=20241120, geometry=geometry, organize_by='field')
+
+data['temp'].plot(figsize=(10, 4), legend=False, c='grey', lw=0.6)
+```
+
+![time_series](https://github.com/CyrilJl/pyisd/blob/main/assets/temp_time_series.png?raw=true)

isd_fetch-0.1.0/isd_fetch.egg-info/PKG-INFO

@@ -0,0 +1,99 @@
+Metadata-Version: 2.1
+Name: isd-fetch
+Version: 0.1.0
+Summary: A Python package for manipulating and analyzing ISD data.
+Author: Cyril Joly
+License: BSD 3-Clause License
+        
+        Copyright (c) 2024, Cyril
+        
+        Redistribution and use in source and binary forms, with or without
+        modification, are permitted provided that the following conditions are met:
+        
+        1. Redistributions of source code must retain the above copyright notice, this
+           list of conditions and the following disclaimer.
+        
+        2. Redistributions in binary form must reproduce the above copyright notice,
+           this list of conditions and the following disclaimer in the documentation
+           and/or other materials provided with the distribution.
+        
+        3. Neither the name of the copyright holder nor the names of its
+           contributors may be used to endorse or promote products derived from
+           this software without specific prior written permission.
+        
+        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+        AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+        IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+        DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+        FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+        DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+        SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+        CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+        OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+        OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+        
+Keywords: isd,noaa
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: geopandas
+Requires-Dist: tqdm
+
+# PyISD: A Python Package for NOAA's ISD Lite Dataset
+
+**PyISD** is a Python package designed for loading and processing NOAA's ISD Lite dataset. The dataset, as described by NOAA, is a streamlined version of the full Integrated Surface Database (ISD). It includes eight common surface parameters in a fixed-width format, free of duplicate values, sub-hourly data, and complicated flags, making it suitable for general research and scientific purposes. For more details, visit the [official ISD homepage](https://www.ncei.noaa.gov/products/land-based-station/integrated-surface-database).
+
+## **Features**
+- Load and query the ISD Lite dataset with ease.
+- Retrieve and process metadata for stations worldwide.
+- Filter data based on spatial and temporal constraints.
+
+## **Example Usage**
+
+### **1. Importing and Loading Metadata**
+You can start by importing the `IsdLite` module, fetching metadata for weather stations worldwide and displaying a sample of the station metadata:
+
+```python
+from pyisd import IsdLite
+
+CRS = 4326
+
+module = IsdLite(crs=CRS, verbose=True)
+module.raw_metadata.sample(5)
+```
+
+The output displays station metadata including station name, latitude, longitude, elevation, and the period of available records:
+
+```
+         USAF   WBAN             STATION NAME CTRY   ST CALL     LAT      LON  \
+8480   377350  99999                   GANDJA   AJ  NaN  NaN  40.717   46.417   
+1023   027710  99999  JOUTSA LEIVONMAKI SAVEN   FI  NaN  NaN  61.883   26.100   
+11880  545340  99999                 TANGSHAN   CH  NaN  NaN  39.650  118.100   
+3795   111900  99999               EISENSTADT   AU  NaN  NaN  47.850   16.533   
+26693  957119  99999     WEST WYALONG AIRPORT   AS  NaN  NaN -33.930  147.200   
+
+        ELEV(M)     BEGIN       END        x       y               geometry  
+8480     309.0  19320101  20241117   46.417  40.717  POINT (46.417 40.717)  
+1023     146.0  20080115  20241112   26.100  61.883    POINT (26.1 61.883)  
+11880     29.0  19560820  20241112  118.100  39.650    POINT (118.1 39.65)  
+3795     189.3  19730627  20241117   16.533  47.850   POINT (16.533 47.85)  
+26693    262.0  19651231  19840629  147.200 -33.930   POINT (147.2 -33.93)  
+```
+
+### **2. Fetching and Visualizing Data**
+To retrieve data, you can specify the time period and spatial constraints. Here, we fetch temperature data (`temp`) for the bounding box around Paris between January 1, 2023, and November 20, 2024:
+
+```python
+from pyisd.misc import get_box
+
+geometry = get_box(place='Paris', width=1., crs=CRS)
+
+data = module.get_data(start=20230101, end=20241120, geometry=geometry, organize_by='field')
+
+data['temp'].plot(figsize=(10, 4), legend=False, c='grey', lw=0.6)
+```
+
+![time_series](https://github.com/CyrilJl/pyisd/blob/main/assets/temp_time_series.png?raw=true)

isd_fetch-0.1.0/isd_fetch.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+LICENSE
+README.md
+pyproject.toml
+isd_fetch.egg-info/PKG-INFO
+isd_fetch.egg-info/SOURCES.txt
+isd_fetch.egg-info/dependency_links.txt
+isd_fetch.egg-info/requires.txt
+isd_fetch.egg-info/top_level.txt
+pyisd/__init__.py
+pyisd/_isd_lite.py
+tests/test_isd_lite.py

isd_fetch-0.1.0/isd_fetch.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

isd_fetch-0.1.0/isd_fetch.egg-info/requires.txt

@@ -0,0 +1,2 @@
+geopandas
+tqdm

isd_fetch-0.1.0/isd_fetch.egg-info/top_level.txt

@@ -0,0 +1 @@
+pyisd

isd_fetch-0.1.0/pyisd/__init__.py

@@ -0,0 +1,3 @@
+from ._isd_lite import IsdLite
+
+__all__ = ['IsdLite']

isd_fetch-0.1.0/pyisd/_isd_lite.py

@@ -0,0 +1,111 @@
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from time import sleep
+from urllib.parse import urljoin
+
+import geopandas as gpd
+import pandas as pd
+from tqdm.auto import tqdm
+
+from .misc import check_params, daterange, proj, to_crs
+
+
+class IsdLite:
+    raw_metadata_url = 'https://www.ncei.noaa.gov/pub/data/noaa/isd-history.txt'
+    data_url = "https://www.ncei.noaa.gov/pub/data/noaa/isd-lite/{year}/"
+    fields = ('temp', 'dewtemp', 'pressure', 'winddirection', 'windspeed', 'skycoverage', 'precipitation-1h', 'precipitation-6h')
+    max_retries = 5
+
+    def __init__(self, crs=4326, verbose=0):
+        self.crs = to_crs(crs)
+        self._get_raw_metadata()
+        self.verbose = verbose
+
+    def _get_raw_metadata(self):
+        for attempt in range(self.max_retries):
+            try:
+                metadata = pd.read_fwf(self.raw_metadata_url, skiprows=19)
+                metadata = metadata.dropna(subset=['LAT', 'LON'])
+                metadata['x'], metadata['y'] = proj(metadata['LON'], metadata['LAT'], 4326, self.crs)
+                self._raw_metadata = gpd.GeoDataFrame(metadata, geometry=gpd.points_from_xy(metadata.x, metadata.y, crs=self.crs))
+            except Exception as e:
+                if attempt < self.max_retries - 1:
+                    sleep(2)
+                else:
+                    raise RuntimeError(f"Failed to download metadata after {self.max_retries} attempts.") from e
+
+    def _filter_metadata(self, geometry):
+        if geometry is None:
+            return self._raw_metadata['USAF'].unique()
+        else:
+            return gpd.clip(self._raw_metadata, geometry)['USAF'].unique()
+
+    @classmethod
+    def _download_read(cls, url):
+        time_features = ['year', 'month', 'day', 'hour']
+        df = pd.read_csv(url, sep='\\s+', header=None, na_values=-9999)
+        df.columns = time_features + list(cls.fields)
+        df[['temp', 'dewtemp', 'pressure', 'windspeed']] /= 10.
+        df.index = pd.to_datetime(df[time_features])
+        df = df.drop(columns=time_features)
+        return df
+
+    @classmethod
+    def _download_data_id(cls, usaf_id, years):
+        ret = []
+        for year in years:
+            try:
+                df = cls._download_read(urljoin(cls.data_url.format(year=year), f'{usaf_id}-99999-{year}.gz'))
+                ret.append(df)
+            except Exception as e:
+                pass
+
+        if ret:
+            return pd.concat(ret)
+        else:
+            return pd.DataFrame()
+
+    def get_data(self, start, end=None, geometry=None, organize_by='location', n_jobs=8):
+        """
+        Fetches weather data from the ISD-Lite dataset for the specified time range and location.
+
+        Args:
+            start (datetime): The start date for the data retrieval.
+            end (datetime, optional): The end date for the data retrieval. If not provided, defaults to the start date.
+            geometry (GeoSeries, optional): A GeoSeries or geometry object to filter stations by spatial location.
+                If None, data for all stations will be retrieved. Defaults to None.
+            organize_by (str, optional): Determines how the resulting data is organized. Options are:
+                - 'location': Organize data by weather station.
+                - 'field': Organize data by weather variable.
+                Defaults to 'location'.
+            n_jobs (int, optional): The number of threads to use for parallel data downloads. Defaults to 8.
+
+        Returns:
+            dict: A dictionary containing the weather data. The structure of the dictionary depends on the
+            `organize_by` parameter:
+                - If 'location': Keys are station IDs, and values are DataFrames with weather data.
+                - If 'field': Keys are weather variables, and values are DataFrames with stations as columns.
+
+        Raises:
+            ValueError: If `organize_by` is not one of the allowed options.
+        """
+        check_params(param=organize_by, params=('field', 'location'))
+        time = daterange(start, end, freq='h')
+        years = time.year.unique()
+        usaf_ids = self._filter_metadata(geometry=geometry)
+
+        def fetch_data(usaf_id):
+            return usaf_id, self._download_data_id(usaf_id=usaf_id, years=years).reindex(index=time)
+
+        ret = {}
+        with ThreadPoolExecutor(max_workers=n_jobs) as executor:
+            futures = {executor.submit(fetch_data, usaf_id): usaf_id for usaf_id in usaf_ids}
+
+            for future in tqdm(as_completed(futures), total=len(futures), disable=(not self.verbose)):
+                usaf_id, data = future.result()
+                if data.size > 0:
+                    ret[usaf_id] = data
+
+        if organize_by == 'field':
+            ret = {field: pd.concat([ret[usaf_id][field].rename(usaf_id) for usaf_id in ret], axis=1) for field in self.fields}
+
+        return ret

isd_fetch-0.1.0/pyproject.toml

@@ -0,0 +1,28 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+packages = ["pyisd"]
+
+[project]
+name = "isd-fetch"
+version = "0.1.0"
+description = "A Python package for manipulating and analyzing ISD data."
+authors = [
+    { name = "Cyril Joly"}
+]
+license = { file = "LICENSE" }
+readme = "README.md"
+keywords = ["isd", "noaa"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+requires-python = ">=3.8"
+
+dependencies = [
+    "geopandas",
+    "tqdm"
+]

isd_fetch-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

isd_fetch-0.1.0/tests/test_isd_lite.py

@@ -0,0 +1,12 @@
+from pyisd import IsdLite
+from pyisd.misc import get_box
+
+
+def test_isdlite():
+    CRS = 4326
+    geometry = get_box(place='Paris', width=1., crs=CRS)
+    module = IsdLite()
+    data = module.get_data(start=20230101, end=20241231, geometry=geometry, organize_by='location')
+    assert data[list(data.keys())[0]].size > 0
+    data = module.get_data(start=20230101, end=20241231, geometry=geometry, organize_by='field')
+    assert data['temp'].size > 0