wxdata 1.6__tar.gz → 1.7__tar.gz

{wxdata-1.6/src/wxdata.egg-info → wxdata-1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wxdata
-Version: 1.6
+Version: 1.7
 Summary: A Python package of end-to-end weather data clients & raw data clients with VPN/PROXY support, data processors that decode variable keys from GRIB format into a plain-language format & various tools for assisting Python automated workflows, querying meteorological datasets and filling gaps in meteorological data.
 Author: Eric J. Drewitz
 Project-URL: Documentation, https://github.com/edrewitz/wxdata/blob/main/Documentation/Landing%20Page.md
@@ -204,6 +204,7 @@ ii) Set up a new environment with an earlier version of Python (must be Python >
 *Advanced Users*
 1) [Using the `client` module to download the latest HadCRUT5 Analysis netCDF file and open this dataset in xarray](https://github.com/edrewitz/WxData-JupyterLab-Examples/blob/main/hadcrut5.ipynb)
 2) [Downloading the GFS0P25 for temperature fields and using run_external_scripts() to post-process this GFS0P25 dataset in an external Python script](https://github.com/edrewitz/WxData-JupyterLab-Examples/blob/main/external_scripts.ipynb)
+3) [Downloading GFS Data Using Byte-Range Requests](https://github.com/edrewitz/WxData-JupyterLab-Examples/blob/main/bytes_range_request.ipynb)
 
 ---------------------------------------------------
 
@@ -340,6 +341,9 @@ ii) Set up a new environment with an earlier version of Python (must be Python >
 ***AWS Open Data***
 1. [AWS Open Data](https://github.com/edrewitz/WxData/blob/main/Documentation/get_open_aws_data.md#get-aws-open-data)
 
+***Byte-Range Requests***
+1. [Byte-Range Requests](https://github.com/edrewitz/WxData/blob/main/Documentation/bytes_range_request.md#bytes-range-request)
+
 ---------------------------------------------------------------
 
 ### Data Querying Tools
@@ -626,6 +630,7 @@ ii) Set up a new environment with an earlier version of Python (must be Python >
       #  - get_excel_data()
       #  - get_xmacis_data()
       #  - get_aws_open_data()
+      #  - byte_range_request()
       import wxdata.client.client as client
 
 -------------------------------------------

{wxdata-1.6 → wxdata-1.7}/README.md

@@ -172,6 +172,7 @@ ii) Set up a new environment with an earlier version of Python (must be Python >
 *Advanced Users*
 1) [Using the `client` module to download the latest HadCRUT5 Analysis netCDF file and open this dataset in xarray](https://github.com/edrewitz/WxData-JupyterLab-Examples/blob/main/hadcrut5.ipynb)
 2) [Downloading the GFS0P25 for temperature fields and using run_external_scripts() to post-process this GFS0P25 dataset in an external Python script](https://github.com/edrewitz/WxData-JupyterLab-Examples/blob/main/external_scripts.ipynb)
+3) [Downloading GFS Data Using Byte-Range Requests](https://github.com/edrewitz/WxData-JupyterLab-Examples/blob/main/bytes_range_request.ipynb)
 
 ---------------------------------------------------
 
@@ -308,6 +309,9 @@ ii) Set up a new environment with an earlier version of Python (must be Python >
 ***AWS Open Data***
 1. [AWS Open Data](https://github.com/edrewitz/WxData/blob/main/Documentation/get_open_aws_data.md#get-aws-open-data)
 
+***Byte-Range Requests***
+1. [Byte-Range Requests](https://github.com/edrewitz/WxData/blob/main/Documentation/bytes_range_request.md#bytes-range-request)
+
 ---------------------------------------------------------------
 
 ### Data Querying Tools
@@ -594,6 +598,7 @@ ii) Set up a new environment with an earlier version of Python (must be Python >
       #  - get_excel_data()
       #  - get_xmacis_data()
       #  - get_aws_open_data()
+      #  - byte_range_request()
       import wxdata.client.client as client
 
 -------------------------------------------

{wxdata-1.6 → wxdata-1.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "wxdata"
-version = "1.6"
+version = "1.7"
 description = "A Python package of end-to-end weather data clients & raw data clients with VPN/PROXY support, data processors that decode variable keys from GRIB format into a plain-language format & various tools for assisting Python automated workflows, querying meteorological datasets and filling gaps in meteorological data."
 readme = "README.md"
 requires-python = ">=3.10"

{wxdata-1.6 → wxdata-1.7}/src/wxdata/__init__.py

@@ -249,4 +249,5 @@ These clients can be easily configured to work on VPN/PROXY connections.
 #  - get_excel_data()
 #  - get_xmacis_data()
 #  - get_aws_open_data()
+#  - byte_range_request()
 import wxdata.client.client as client

wxdata-1.7/src/wxdata/client/byte_range.py

@@ -0,0 +1,204 @@
+"""
+This file hosts the function that fetches the byte-range for a variable.
+
+(C) Eric J. Drewitz 2025-2026
+"""
+
+import requests
+
+from tqdm.auto import tqdm
+
+
+def fetch_range(url, 
+                start, 
+                end,
+                proxies):
+    
+    """
+    This function downloads the data corresponding to the bytes-range for each variable
+    
+    Required Arguments:
+    
+    1) url (String) - The URL of the GRIB file that will be downloaded.
+    
+    2) start (Integer) - The starting byte.
+    
+    3) end (Integer) - The ending byte.
+    
+    4) proxies (dict or None) - Default=None. If the user is using proxy server(s), the user must change the following:
+
+       proxies=None ---> proxies={
+                               'http':'http://your-proxy-address:port',
+                               'https':'http://your-proxy-address:port'
+                               }
+                               
+    Optional Arguments: None
+    
+    Returns
+    -------
+    
+    The data to be written to a GRIB file.
+    """
+    
+    headers = {"Range": f"bytes={start}-{end}" if end else f"bytes={start}-"}
+    
+    if proxies == None:
+        response = requests.get(url, 
+                         headers=headers,
+                         stream=True)
+    else:
+        response = requests.get(url, 
+                         headers=headers,
+                         proxies=proxies,
+                         stream=True)
+    
+    response.raise_for_status()
+    response.close()
+    
+    if len(response.content) == 0:
+        if proxies == None:
+            response = requests.get(url, 
+                            headers=headers)
+        else:
+            response = requests.get(url, 
+                            headers=headers,
+                            proxies=proxies)
+    else:
+        pass
+    
+    return response.content
+
+
+def fetch_data(ranges,
+                url, 
+                start, 
+                end,
+                proxies):
+    
+    """
+    This function downloads the data corresponding to the bytes-range for each variable
+    
+    Required Arguments:
+    
+    1) url (String) - The URL of the GRIB file that will be downloaded.
+    
+    2) start (Integer) - The starting byte.
+    
+    3) end (Integer) - The ending byte.
+    
+    4) proxies (dict or None) - Default=None. If the user is using proxy server(s), the user must change the following:
+
+       proxies=None ---> proxies={
+                               'http':'http://your-proxy-address:port',
+                               'https':'http://your-proxy-address:port'
+                               }
+                               
+    Optional Arguments: None
+    
+    Returns
+    -------
+    
+    The data to be written to a GRIB file.
+    """
+    
+    results = {}
+    for (v, l), (start, end) in ranges.items():
+        results[(v, l)] = fetch_range(
+            url,
+            start,
+            end,
+            proxies
+        )
+
+    return results
+
+
+def download_grib_data_by_byte_range(ranges,
+                             chunk_size,
+                             path,
+                             filename,
+                             grib_url,
+                             start,
+                             end,
+                             proxies):
+    """
+    results: dict[(var, level)] = bytes
+    Writes all GRIB messages to a single file with polished progress bars.
+    
+    This function writes the GRIB file with a progress bar. 
+    This is a progress bar for the byte-range requests. 
+    
+    Required Arguments:
+    
+    1) ranges (Tuple List) - A tuple list of byte-ranges.
+    
+    2) chunk_size (Integer) - The size of the chunks when writing the GRIB/NETCDF data to a file.
+    
+    3) path (String) - The directory where the file is saved to. 
+    
+    4) filename (String) - The name the user wishes to save the file as. 
+    
+    5) grib_url (String) - The URL of the GRIB file that will be downloaded.
+    
+    6) start (Integer) - The starting byte.
+    
+    7) end (Integer) - The ending byte.
+    
+    8) proxies (dict or None) - Default=None. If the user is using proxy server(s), the user must change the following:
+
+       proxies=None ---> proxies={
+                               'http':'http://your-proxy-address:port',
+                               'https':'http://your-proxy-address:port'
+                               }
+    
+    Optional Arguments: None
+    
+    Returns
+    -------
+    
+    A GRIB file of {filename} saved to {path}
+    """
+    
+    results = fetch_data(ranges,
+                            grib_url, 
+                            start, 
+                            end,
+                            proxies)
+
+    ordered = sorted(results.items(), key=lambda x: (x[0][0], x[0][1]))
+
+    # Master progress bar (stays visible)
+    master = tqdm(
+        total=len(ordered),
+        desc="Writing GRIB messages",
+        position=0,
+        leave=False,
+        bar_format="{desc:<25} |{bar:30}| {n}/{total}"
+    )
+
+    with open(f"{path}/{filename}", "wb") as f:
+        for (var, level), data in ordered:
+
+            total_bytes = len(data)
+            label = f"{var} @ {level}"
+
+            # Inner bar (disappears after finishing)
+            with tqdm(
+                total=total_bytes,
+                unit="B",
+                unit_scale=True,
+                unit_divisor=chunk_size,
+                desc=label,
+                position=1,
+                leave=False,
+                bar_format="{desc:<30} |{bar:30}| {percentage:3.0f}% [{n_fmt}/{total_fmt}]"
+            ) as pbar:
+
+                for i in range(0, total_bytes, chunk_size):
+                    chunk = data[i:i+chunk_size]
+                    f.write(chunk)
+                    pbar.update(len(chunk))
+
+            master.update(1)
+
+    master.close()

{wxdata-1.6 → wxdata-1.7}/src/wxdata/client/client.py

@@ -8,10 +8,12 @@ These functions are compatible with users on VPN/PROXY connections as well as no
 3) get_excel_data
 4) get_xmacis_data
 5) get_open_aws_data
+6) byte_range_request
 
 (C) Eric J. Drewitz 2025-2026
 """
 
+import itertools as _itertools
 import requests as _requests
 import time as _time
 import sys as _sys
@@ -27,8 +29,11 @@ from datetime import(
     timedelta as _timedelta
 )
 
-from wxdata.utils.progress_bar import progress_bar as _progress_bar
+from wxdata.utils.nomads_gribfilter import key_list as _key_list
+from wxdata.client.level_coords import get_level_expression as _get_level_expression
 from wxdata.utils.xmacis2_cleanup import clean_pandas_dataframe as _clean_pandas_dataframe
+from wxdata.utils.progress_bar import progress_bar as _progress_bar
+from wxdata.client.byte_range import download_grib_data_by_byte_range as _download_grib_data_by_byte_range
 from wxdata.utils.recycle_bin import(
     clear_recycle_bin_windows as _clear_recycle_bin_windows,
     clear_trash_bin_mac as _clear_trash_bin_mac,
@@ -802,4 +807,165 @@ def get_aws_open_data(url,
         finally:
             if r:
                 r.close() # Ensure the connection is closed.
+                
+                
+def byte_range_request(grib_url,
+                      idx_url,
+                      variables,
+                      levels,
+                      level_type,
+                      path,
+                      filename,
+                      proxies=None,
+                      chunk_size=1024,
+                      notifications='on',
+                      clear_recycle_bin=False):
+    
+    """
+    This client downloads GRIB data for a specific variable that is defined by the range in bytes in that GRIB file. 
+    This is useful when the user wants to download a GRIB file where there is no GRIB filter present, especially when the file size is large.
+    This will allow users to download the variable they are interested in and filter out all other variables prior to downloading.
+    
+    Required Arguments:
+    
+    1) grib_url (String) - The URL of the GRIB file that will be downloaded.
+    
+    2) idx_url (String) - The URL of the index file that corresponds to the GRIB file (ends in .idx).
+    
+    3) variables (String List) - The list of variables to be downloaded.
+    
+    4) levels (Float or Integer List) - The list of pressure or height levels. 
+    
+    5) level_type (String) - The type of level.
+    
+        Level Types
+        -----------
+        
+        'hybrid'
+        'entire atmosphere'
+        'surface':'surface',
+        'boundary layer'
+        'pressure'
+        'mean sea level'
+        'height above ground'
+        'height below ground'
+        'height above sea level'
+        'entire atmosphere single layer'
+        'low cloud layer'
+        'middle cloud layer'
+        'high cloud layer'
+        'cloud ceiling'
+        'tropopause'
+        'max wind'
+        'isothermal'
+        'highest tropospheric freezing level'
+        'sigma layer'
+        'sigma level'
+        'potential vorticity surface'
+        'reserved'
+    
+    6) path (String) - The directory where the file is saved to. 
+    
+    7) filename (String) - The name the user wishes to save the file as. 
+    
+    Optional Arguments:
+    
+    1) proxies (dict or None) - Default=None. If the user is using proxy server(s), the user must change the following:
+
+       proxies=None ---> proxies={
+                               'http':'http://your-proxy-address:port',
+                               'https':'http://your-proxy-address:port'
+                               }
+                        
+    2) chunk_size (Integer) - Default=8192. The size of the chunks when writing the GRIB/NETCDF data to a file.
+    
+    3) notifications (String) - Default='on'. Notification when a file is downloaded and saved to {path}
+    
+    4) clear_recycle_bin (Boolean) - (Default=False in WxData >= 1.2.5) (Default=True in WxData < 1.2.5). When set to True, 
+        the contents in your recycle/trash bin will be deleted with each run of the program you are calling WxData. 
+        This setting is to help preserve memory on the machine. 
+        
+    Returns
+    -------
+    
+    Downloads a partial GRIB file consisting of the variable the user specifies.     
+    """
+    if clear_recycle_bin == True:
+        _clear_recycle_bin_windows()
+        _clear_trash_bin_mac()
+        _clear_trash_bin_linux()
+    else:
+        pass
+    
+    try:
+        _os.makedirs(f"{path}")
+    except Exception as e:
+        pass
+    
+    variables = _key_list(variables)
+    
+    if proxies == None:
+        idx_text = _requests.get(idx_url).text
+    else:
+        idx_text = _requests.get(idx_url, proxies=proxies).text
+        
+    records = []
+    for line in idx_text.strip().splitlines():
+        parts = line.split(':')
+        msg_no = int(parts[0])
+        offset = int(parts[1])
+        var = parts[3]
+        lev = parts[4]
+        records.append({
+            "msg": msg_no,
+            "offset": offset,
+            "var": var,
+            "lev": lev
+        })
+        
+    req_levels, levels = _get_level_expression(levels,
+                                level_type)
+    
+    if levels is not None:
+        reqs = list(_itertools.product(variables, req_levels))
+    else:
+        reqs = []
+        for v in variables:
+            req = (v, req_levels)
+            reqs.append(req)
+            
+    ranges = {}
+    for v, l in reqs:
+        matches = [r for r in records if r["var"] == v and r["lev"] == l]
+        if not matches:
+            print(f"{v} is not a valid variable OR {l} is not a valid level.")
+            print(f"Please visit {idx_url} to look at the variables in the GRIB file meta-data")
+            _sys.exit(1)
+        else:
+            pass
+
+        rec = matches[0]
+        start = rec["offset"]
+
+        idx = records.index(rec)
+        if idx < len(records) - 1:
+            end = records[idx + 1]["offset"] - 1
+        else:
+            end = None
+
+        ranges[(v, l)] = (start, end)
+    
+    _download_grib_data_by_byte_range(ranges,
+                             chunk_size,
+                             path,
+                             filename,
+                             grib_url,
+                             start,
+                             end,
+                             proxies)
+    
+    if notifications == 'on':
+        print(f"{filename} saved to {path}")
+        
+    
         

wxdata-1.7/src/wxdata/client/level_coords.py

@@ -0,0 +1,141 @@
+"""
+This file hosts the functions that handle the level coordinates for byte-range requests.
+
+(C) Eric J. Drewitz 2025-2026
+"""
+
+def get_level_suffix(level_type):
+    
+    """
+    This function returns the level type suffix.
+    
+    Required Arguments:
+    
+    1) level_type (String) - The type of level the coordinates are on.
+    
+    Level Types
+    -----------
+    
+    'hybrid'
+    'entire atmosphere'
+    'surface':'surface',
+    'boundary layer'
+    'pressure'
+    'mean sea level'
+    'height above ground'
+    'height below ground'
+    'height above sea level'
+    'entire atmosphere single layer'
+    'low cloud layer'
+    'middle cloud layer'
+    'high cloud layer'
+    'cloud ceiling'
+    'tropopause'
+    'max wind'
+    'isothermal'
+    'highest tropospheric freezing level'
+    'sigma layer'
+    'sigma level'
+    'potential vorticity surface'
+    'reserved'
+    
+    Optional Arguments: None
+    
+    Returns
+    -------
+    
+    The level type suffix.    
+    """
+    
+    level_types = {
+        
+        'hybrid':'hybrid level',
+        'entire atmosphere':'entire atmosphere',
+        'surface':'surface',
+        'boundary layer':'planetary boundary layer',
+        'pressure':'mb',
+        'mean sea level':'mean sea level',
+        'height above ground':'m above ground',
+        'height below ground':'m below ground',
+        'height above sea level':'m above mean sea level',
+        'entire atmosphere single layer':'entire atmosphere (considered as a single layer)',
+        'low cloud layer':'low cloud layer',
+        'middle cloud layer':'middle cloud layer',
+        'high cloud layer':'high cloud layer',
+        'cloud ceiling':'cloud ceiling',
+        'tropopause':'tropopause',
+        'max wind':'max wind',
+        'isothermal':'0C isotherm',
+        'highest tropospheric freezing level':'highest tropospheric freezing level',
+        'sigma layer':'sigma layer',
+        'sigma level':'sigma level',
+        'potential vorticity surface':'PV=2e-06 (Km^2/kg/s) surface',
+        'reserved':'reserved'
+    }
+    
+    return level_types[level_type]
+
+
+def get_level_expression(levels,
+                         level_type):
+    
+    """
+    This function returns the full expression for the level.
+    
+    Required Arguments:
+    
+    1) levels (Float or Integer List) - The pressure or height levels corresponding to level type. 
+        (i.e. [700, 500, 250] if level type is 'pressure' or [2, 10] if level type is 'height above ground')
+    
+    2) level_type (String) - The type of level the coordinates are on.
+    
+    Level Types
+    -----------
+    
+    'hybrid'
+    'entire atmosphere'
+    'surface':'surface',
+    'boundary layer'
+    'pressure'
+    'mean sea level'
+    'height above ground'
+    'height below ground'
+    'height above sea level'
+    'entire atmosphere single layer'
+    'low cloud layer'
+    'middle cloud layer'
+    'high cloud layer'
+    'cloud ceiling'
+    'tropopause'
+    'max wind'
+    'isothermal'
+    'highest tropospheric freezing level'
+    'sigma layer'
+    'sigma level'
+    'potential vorticity surface'
+    'reserved'
+    
+    Optional Arguments: None
+    
+    Returns
+    -------
+    
+    The full expression for the level to make the bytes-range request.
+    """
+    level_type = level_type.lower()
+
+    suffix = get_level_suffix(level_type)    
+    if levels is not None:
+        expression = []
+        for level in levels:
+            ex = f"{level} {suffix}"
+            expression.append(ex)
+            levels = levels
+    else:
+        expression = suffix
+        levels = None
+        
+    return expression, levels
+    
+    
+    

{wxdata-1.6 → wxdata-1.7}/src/wxdata/noaa/nws.py

@@ -199,7 +199,7 @@ def _FIX_1D_GRIB_DATA(ds_short,
         
     dims = ("step", "latitude", "longitude")
     short_coords = {
-        "step": len(ds_list_short),
+        "step": ds_short['step'],
         "latitude": lat1d,  
         "longitude": lon1d,  
     }
@@ -239,7 +239,7 @@ def _FIX_1D_GRIB_DATA(ds_short,
         
     dims = ("step", "latitude", "longitude")
     extended_coords = {
-        "step": len(ds_list_extended),
+        "step": ds_extended['step'],
         "latitude": lat1d,  
         "longitude": lon1d,  
     }

{wxdata-1.6 → wxdata-1.7}/src/wxdata/rtma/process.py

@@ -317,4 +317,4 @@ def process_rtma_data(path,
     else:
     
         ds = shift_longitude(ds)
-        return ds
+        return ds

{wxdata-1.6 → wxdata-1.7}/src/wxdata/rtma/rtma.py

@@ -517,4 +517,4 @@ def rtma_comparison(model='rtma',
         
     
     
-    
+    

{wxdata-1.6 → wxdata-1.7}/src/wxdata/utils/progress_bar.py

@@ -4,10 +4,7 @@ This file hosts a function that displays a progress bar for downloading a file.
 (C) Eric J. Drewitz 2025-2026
 """
 
-
-import os
-
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 def progress_bar(response,
                  path,
@@ -41,4 +38,4 @@ def progress_bar(response,
         with open(f"{path}/{filename}", "wb") as file:
             for data in response.iter_content(blocksize):
                 file.write(data)
-                bar.update(len(data))
+                bar.update(len(data))

{wxdata-1.6 → wxdata-1.7/src/wxdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wxdata
-Version: 1.6
+Version: 1.7
 Summary: A Python package of end-to-end weather data clients & raw data clients with VPN/PROXY support, data processors that decode variable keys from GRIB format into a plain-language format & various tools for assisting Python automated workflows, querying meteorological datasets and filling gaps in meteorological data.
 Author: Eric J. Drewitz
 Project-URL: Documentation, https://github.com/edrewitz/wxdata/blob/main/Documentation/Landing%20Page.md
@@ -204,6 +204,7 @@ ii) Set up a new environment with an earlier version of Python (must be Python >
 *Advanced Users*
 1) [Using the `client` module to download the latest HadCRUT5 Analysis netCDF file and open this dataset in xarray](https://github.com/edrewitz/WxData-JupyterLab-Examples/blob/main/hadcrut5.ipynb)
 2) [Downloading the GFS0P25 for temperature fields and using run_external_scripts() to post-process this GFS0P25 dataset in an external Python script](https://github.com/edrewitz/WxData-JupyterLab-Examples/blob/main/external_scripts.ipynb)
+3) [Downloading GFS Data Using Byte-Range Requests](https://github.com/edrewitz/WxData-JupyterLab-Examples/blob/main/bytes_range_request.ipynb)
 
 ---------------------------------------------------
 
@@ -340,6 +341,9 @@ ii) Set up a new environment with an earlier version of Python (must be Python >
 ***AWS Open Data***
 1. [AWS Open Data](https://github.com/edrewitz/WxData/blob/main/Documentation/get_open_aws_data.md#get-aws-open-data)
 
+***Byte-Range Requests***
+1. [Byte-Range Requests](https://github.com/edrewitz/WxData/blob/main/Documentation/bytes_range_request.md#bytes-range-request)
+
 ---------------------------------------------------------------
 
 ### Data Querying Tools
@@ -626,6 +630,7 @@ ii) Set up a new environment with an earlier version of Python (must be Python >
       #  - get_excel_data()
       #  - get_xmacis_data()
       #  - get_aws_open_data()
+      #  - byte_range_request()
       import wxdata.client.client as client
 
 -------------------------------------------

{wxdata-1.6 → wxdata-1.7}/src/wxdata.egg-info/SOURCES.txt

@@ -25,7 +25,9 @@ src/wxdata/cfs/cfs.py
 src/wxdata/cfs/file_scanner.py
 src/wxdata/cfs/url_scanners.py
 src/wxdata/client/__init__.py
+src/wxdata/client/byte_range.py
 src/wxdata/client/client.py
+src/wxdata/client/level_coords.py
 src/wxdata/ecmwf/__init__.py
 src/wxdata/ecmwf/ecmwf.py
 src/wxdata/ecmwf/file_funcs.py