grib2sail 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

grib2sail/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.1.0'
-__version_tuple__ = version_tuple = (0, 1, 0)
+__version__ = version = '0.2.0'
+__version_tuple__ = version_tuple = (0, 2, 0)
 
 __commit_id__ = commit_id = None

grib2sail/downloader.py

@@ -1,102 +1,54 @@
-from rich.progress import Progress
-from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import threading
 import requests
-import re
+from rich.progress import Progress
 
 from grib2sail.logger import logger
+from grib2sail.downloader_arom import handle_fetch_error_arom, download_arom
 import grib2sail.variables as v
-from grib2sail.token import get_arome_token
+
+thread_local = threading.local()
+
+def get_session():
+  if not hasattr(thread_local, 'session'):
+    thread_local.session = requests.Session()
+  return thread_local.session
 
 def download_gribs(m, s, d, lat, lon):
   if m.startswith('arome'):
-    download_arome(m, s, d, lat, lon)
+    download_arom(m, s, d, lat, lon)
   else:
     logger.error_exit(f"Downloader failed: unexpected model: {m}")
 
-def download_arome(model, step, data, lat, lon):
-  token = get_arome_token()
-  logger.debug('Token for AROME API retrieived')
-  
-  # Coverages list all the individual layers categories to download
-  coverages = []
-  if v.DATAS[0] in data:
-    coverages += [v.AROM_DATAS['wind_u'], v.AROM_DATAS['wind_v']]
-  if v.DATAS[1] in data:
-    coverages += [v.AROM_DATAS['wind_gust']]
-  if v.DATAS[2] in data:
-    coverages += [v.AROM_DATAS['pressure']]
-  if v.DATAS[3] in data:
-    coverages += [v.AROM_DATAS['cloud']]
-  
-  # Get latest available forecast date from arome /GetCapabilities api endpoint
+def get_layers(model, urls, header):
+  # Downloading every layers
+  layers = [None] * len(urls)
+  with Progress() as progress:
+    # Showing a progress bar
+    task = progress.add_task('Downloading layers...', total=len(urls))
+
+    # Downloading the layer
+    with ThreadPoolExecutor(max_workers=10) as executor:
+      futures = [
+        executor.submit(fetch, i, url, header, model)
+        for i, url in enumerate(urls)
+      ]
+
+      for future in as_completed(futures):
+        idx, layer = future.result()
+        layers[idx] = layer
+        progress.advance(task)
+  return layers
+
+def fetch(idx, url, headers, model):
   try:
-    capa = requests.get(
-      v.AROM_URLS[f"{model}_capa"], 
-      headers = {'Authorization': f"Bearer {token}"},
-      timeout = 60,
-    )
+    session = get_session()
+    r = session.get(url, headers=headers,timeout = 60)
+    r.raise_for_status()
+    return idx, r.content
   except Exception as e:
-    logger.error_exit(f"Failed to contact METEO FRANCE servers: {e}")
-  
-  # Parse the GetCapabilities XML response to find the latest available coverage
-  lines = [line for line in capa.text.splitlines() if coverages[0] in line]
-  if lines:
-    # Forecast available dates look like 1970-01-01T00:00:00Z
-    # The last line holds the lastest available forecast run
-    latestRun = re.search(
-      r"\d{4}-\d{2}-\d{2}T\d{2}\.\d{2}\.\d{2}Z",
-      lines[-1]
-    )
-    if latestRun:
-      latestRun = latestRun.group()
+    if model in v.MODELS[:2]:
+      handle_fetch_error_arom(e)
     else:
-      msg = "Error fetching AROM capabilities, couldn't find latest date"
-      logger.error_exit(msg)
-  else:
-    msg = "Error fetching AROM capabilities, couldn't find latest run"
-    logger.error_exit(msg)
-    
-  # Download all layers as individual grib files into one output file
-  file = Path(f"{model}_{latestRun}_{step}.grib2")
-  file.unlink(missing_ok=True)
-  with open(file, "ab") as outfile, Progress() as progress:
-    # Select forecast prevision time based on user input
-    # 3600 means layer is the prevision for 1h after latestRun
-    times = list(range(
-      int(step[:-1]) * 3600, 
-      172800+1, 
-      int(step[:-1]) * 3600)
-    )
-    logger.debug(f"forecast to downloads are {times}")
-    # Showing a progress bar
-    task = progress.add_task(
-      'Downloading layers...', 
-      total = len(coverages) * len(times)
-    )
-    for coverage in coverages:
-      for time in times:
-        paramCovId = f"&coverageid={coverage}{latestRun}"
-        subTime = f"&subset=time({time})"
-        subLat = f"&subset=lat({lat[0]},{lat[1]})"
-        subLon = f"&subset=long({lon[0]},{lon[1]})"
-        if 'SPECIFIC_HEIGHT' in coverage:
-          subHeight = '&subset=height(10)'
-        else:
-          subHeight = ''
-        paramSubset = subTime + subLat + subLon + subHeight
-        url=v.AROM_URLS[f"{model}_cov"]+ paramCovId + paramSubset
-        logger.debug(f"Downloading {url}")
-        try:
-          r = requests.get(
-            url, 
-            headers = {'Authorization': f"Bearer {token}"},
-            timeout = 60)
-          r.raise_for_status()
-          outfile.write(r.content)
-        except requests.exceptions.HTTPError:
-          logger.warning(
-            f"Failed to download {url} status code is {r.status_code}"
-          )
-        except Exception as e:
-          logger.error_exit(f"Download failed: {e}", to_clean=[file])
-        progress.update(task, advance=1)
+      logger.error_exit(f"Download failed: {e}")
+    return idx, None

grib2sail/downloader_arom.py

@@ -0,0 +1,115 @@
+from pathlib import Path
+import re
+import requests
+import time as t
+
+import grib2sail.variables as v
+import grib2sail.variables_arom as va
+import grib2sail.downloader as d
+from grib2sail.logger import logger
+from grib2sail.token import get_arome_token
+
+def download_arom(model, step, data, lat, lon):
+  token = get_arome_token()
+  
+  # Coverages list all the individual layers categories to download
+  coverages = []
+  if v.DATAS[0] in data:
+    coverages += [va.AROM_DATAS['wind_u'], va.AROM_DATAS['wind_v']]
+  if v.DATAS[1] in data:
+    coverages += [va.AROM_DATAS['wind_gust']]
+  if v.DATAS[2] in data:
+    coverages += [va.AROM_DATAS['pressure']]
+  if v.DATAS[3] in data:
+    coverages += [va.AROM_DATAS['cloud']]
+  
+  # Get latest available forecast date from arome /GetCapabilities api endpoint
+  logger.info('Finding latest available forecast')
+  session = d.get_session()
+  try:
+    capa = session.get(
+      va.AROM_URLS[f"{model}_capa"], 
+      headers = {'Authorization': f"Bearer {token}"},
+      timeout = 60,
+    )
+  except Exception as e:
+    logger.error_exit(f"Failed to contact METEO FRANCE servers: {e}")
+  
+  # Parse the GetCapabilities XML response to find the latest available coverage
+  lines = [line for line in capa.text.splitlines() if coverages[0] in line]
+  if lines:
+    # Forecast available dates look like 1970-01-01T00:00:00Z
+    # The last line holds the lastest available forecast run
+    latestRun = re.search(
+      r"\d{4}-\d{2}-\d{2}T\d{2}\.\d{2}\.\d{2}Z",
+      lines[-1]
+    )
+    if latestRun:
+      latestRun = latestRun.group()
+    else:
+      msg = "Error fetching AROM capabilities, couldn't find latest date"
+      logger.error_exit(msg)
+  else:
+    msg = "Error fetching AROM capabilities, couldn't find latest run"
+    logger.error_exit(msg)
+
+  # Select forecast prevision time based on user input
+  # 3600 means layer is the prevision for 1h after latestRun
+  times = list(range(
+    int(step[:-1]) * 3600,
+    172800+1,
+    int(step[:-1]) * 3600)
+  )
+  logger.debug(f"Forecast to download are {times}")
+
+  # Generating the urls to retreive requested layers
+  header = {'Authorization': f"Bearer {token}"}
+  urls = []
+  for coverage in coverages:
+    for time in times:
+      paramCovId = f"&coverageid={coverage}{latestRun}"
+      subTime = f"&subset=time({time})"
+      subLat = f"&subset=lat({lat[0]},{lat[1]})"
+      subLon = f"&subset=long({lon[0]},{lon[1]})"
+      if 'SPECIFIC_HEIGHT' in coverage:
+        subHeight = '&subset=height(10)'
+      else:
+        subHeight = ''
+      paramSubset = subTime + subLat + subLon + subHeight
+      urls.append(va.AROM_URLS[f"{model}_cov"]+ paramCovId + paramSubset)
+
+  # Downloading the layers
+  layers = []
+  if len(urls) < 100:
+    layers = d.get_layers(model, urls, header)
+  else:
+    msg = f"The requested grib has {len(urls)} layers, but MeteoFrance"
+    msg += ' servers limit requests to 100 per minute. This program will'
+    msg += ' sleep 1 minute every 100 layer util the complete grib file'
+    msg += ' is downloaded. You might want to consider reducing the number'
+    msg += ' of layers by increasing the step or reducing the number of'
+    msg += ' data'
+    logger.warning(msg)
+    for i in range(0, len(urls), 100):
+      layers.extend(d.get_layers(model, urls[i:i+100], header))
+      if i+100 < len(urls):
+        logger.info('Sleeping 1 minute...')
+        t.sleep(60)
+  
+  # Output the file once all the layers have been downloaded
+  file = Path(f"{model}_{latestRun}_{step}.grib2")
+  file.unlink(missing_ok=True)
+  with open(file, "wb") as outfile:
+    for layer in layers:
+      if layer:
+        outfile.write(layer)
+
+def handle_fetch_error_arom(e):
+  if isinstance(e, requests.exceptions.HTTPError):
+    url = e.response.url
+    layer = re.search(r"coverageid=(.*?)__", url).group(1)
+    time = int(re.search(r"subset=time\(([^()]*)", url).group(1)) / 3600
+    logger.warning(f"Missing layer: {layer} at time: {int(time)}h")
+    logger.debug(f"Error was {e}")
+  else:
+    logger.error_exit(f"Download failed: {e}")

grib2sail/token.py

@@ -1,16 +1,18 @@
 import os
 import keyring
 import getpass
-import requests
 
-from grib2sail import variables as v 
+import grib2sail.downloader as d
+from grib2sail import variables_arom as va
 from grib2sail.logger import logger
 
 def get_arome_token():
+  logger.info('Authenticating to MeteoFrance')
   appId = get_arome_appid()
+  session = d.get_session()
   try:
-    response = requests.post(
-      v.AROM_URLS['token'], 
+    response = session.post(
+      va.AROM_URLS['token'], 
       data = { 'grant_type': 'client_credentials' }, 
       headers = { 'Authorization': f"Basic {appId}" },
       timeout = 60,

grib2sail/variables.py

@@ -1,23 +1,3 @@
 MODELS = ['arome_antilles', 'arome001']
 STEPS = ['1h', '3h', '6h', '12h']
 DATAS = ['wind', 'wind_gust', 'pressure', 'cloud', 'rain']
-
-AROM_DATAS = {
-  'wind_u': 'U_COMPONENT_OF_WIND__SPECIFIC_HEIGHT_LEVEL_ABOVE_GROUND___',
-  'wind_v': 'V_COMPONENT_OF_WIND__SPECIFIC_HEIGHT_LEVEL_ABOVE_GROUND___',
-  'wind_gust': 'WIND_SPEED_GUST__SPECIFIC_HEIGHT_LEVEL_ABOVE_GROUND___',
-  'pressure': 'PRESSURE__MEAN_SEA_LEVEL___',
-  'cloud': 'TOTAL_CLOUD_COVER__GROUND_OR_WATER_SURFACE___'
-}
-
-AROM_URLS = {
-  'token': 'https://portail-api.meteofrance.fr/token',
-  f"{MODELS[0]}_cov": 'https://public-api.meteofrance.fr/public/arome/1.0/wcs/MF-NWP-HIGHRES-AROME-OM-0025-ANTIL-WCS/GetCoverage?service=WCS&version=2.0.1&format=application/wmo-grib',
-  f"{MODELS[1]}_cov": 'https://public-api.meteofrance.fr/public/arome/1.0/wcs/MF-NWP-HIGHRES-AROME-001-FRANCE-WCS/GetCoverage?service=WCS&version=2.0.1&format=application/wmo-grib',
-  f"{MODELS[0]}_capa": 'https://public-api.meteofrance.fr/public/arome/1.0/wcs/MF-NWP-HIGHRES-AROME-OM-0025-ANTIL-WCS/GetCapabilities?service=WCS&version=1.3.0&language=eng',
-  f"{MODELS[1]}_capa": 'https://public-api.meteofrance.fr/public/arome/1.0/wcs/MF-NWP-HIGHRES-AROME-001-FRANCE-WCS/GetCapabilities?service=WCS&version=1.3.0&language=eng',
-}
-
-
-
-

grib2sail/variables_arom.py

@@ -0,0 +1,17 @@
+import grib2sail.variables as v
+
+AROM_DATAS = {
+  'wind_u': 'U_COMPONENT_OF_WIND__SPECIFIC_HEIGHT_LEVEL_ABOVE_GROUND___',
+  'wind_v': 'V_COMPONENT_OF_WIND__SPECIFIC_HEIGHT_LEVEL_ABOVE_GROUND___',
+  'wind_gust': 'WIND_SPEED_GUST__SPECIFIC_HEIGHT_LEVEL_ABOVE_GROUND___',
+  'pressure': 'PRESSURE__MEAN_SEA_LEVEL___',
+  'cloud': 'TOTAL_CLOUD_COVER__GROUND_OR_WATER_SURFACE___'
+}
+
+AROM_URLS = {
+  'token': 'https://portail-api.meteofrance.fr/token',
+  f"{v.MODELS[0]}_cov": 'https://public-api.meteofrance.fr/public/arome/1.0/wcs/MF-NWP-HIGHRES-AROME-OM-0025-ANTIL-WCS/GetCoverage?service=WCS&version=2.0.1&format=application/wmo-grib',
+  f"{v.MODELS[1]}_cov": 'https://public-api.meteofrance.fr/public/arome/1.0/wcs/MF-NWP-HIGHRES-AROME-001-FRANCE-WCS/GetCoverage?service=WCS&version=2.0.1&format=application/wmo-grib',
+  f"{v.MODELS[0]}_capa": 'https://public-api.meteofrance.fr/public/arome/1.0/wcs/MF-NWP-HIGHRES-AROME-OM-0025-ANTIL-WCS/GetCapabilities?service=WCS&version=1.3.0&language=eng',
+  f"{v.MODELS[1]}_capa": 'https://public-api.meteofrance.fr/public/arome/1.0/wcs/MF-NWP-HIGHRES-AROME-001-FRANCE-WCS/GetCapabilities?service=WCS&version=1.3.0&language=eng',
+}

{grib2sail-0.1.0.dist-info → grib2sail-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: grib2sail
-Version: 0.1.0
+Version: 0.2.0
 Summary: Grib files downloader for sailing purposes
 Author-email: Chinkara <poubelledechinkara@outlook.com>
 License-Expression: GPL-3.0-or-later
@@ -17,7 +17,7 @@ Dynamic: license-file
 # GRIB2Sail
 
 ![PyPI version](https://img.shields.io/pypi/v/grib2sail.svg)
-![CI](https://img.shields.io/github/workflow/status/Ch1nkara/GRIB2Sail/CI)
+![CI](https://img.shields.io/github/actions/workflow/status/Ch1nkara/GRIB2Sail/release.yml)
 ![License](https://img.shields.io/badge/license-GPL%20v3-blue.svg)
 
 Grib files downloader for sailing purposes.
@@ -37,7 +37,7 @@ To download GRIB from meteofrance's models (Aome), you must create a free
 account on meteofrance.fr. The procedure is as follow:
  1. Create an account on [the Météo-France API portal](https://portail-api.meteofrance.fr)
  2. Subscribe to the desired service (Arome)
- 3. Go to "My API" then, from your subscribe model: "Generate Token"
+ 3. Go to "My API" then, from your subscribed model: "Generate Token"
  4. Checkout the curl field at the bottom, it looks like :
    ```bash
    curl -k -X POST https://portal-api.meteofrance.fr/token -d "grant_type=client_credentials" -H "Authorization: Basic ABCDEF1234abcdef"

grib2sail-0.2.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+grib2sail/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+grib2sail/__main__.py,sha256=U5PTpeJRtXGpcO8YPUdCD2hORF8T68IM6JW1-dDyYqE,37
+grib2sail/_version.py,sha256=Dg8AmJomLVpjKL6prJylOONZAPRtB86LOce7dorQS_A,704
+grib2sail/cli.py,sha256=3IcGISWNF9XpDhSdSqb6z5bnc2YFmKlQKtTPH0M7hzI,2247
+grib2sail/downloader.py,sha256=ZQDaPabDuKlb7CqGxwabI73T5pKC_jVdqcCyT6A6ch4,1566
+grib2sail/downloader_arom.py,sha256=FA9IDvhcBiEY7uJuHTcuLY-Ku4US5uPV5wLks3maNuw,4038
+grib2sail/logger.py,sha256=_erYrZlKPjhDu-reFkyylodFgSVtOcXJg5F1drYeQNY,456
+grib2sail/token.py,sha256=gNYCBNEGmjMb_RGq79wvtflhKxqiiX4GXpqfMR3eiJ0,1443
+grib2sail/variables.py,sha256=M0ZcusbURgPuI1ZbRjnCqRr1Ke-2057cz3EycCHYB2E,133
+grib2sail/variables_arom.py,sha256=RNJZT7O5EFa4HtVYrUuH7EHwxZ9u_63F1vkBUekfOfo,1169
+grib2sail-0.2.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+grib2sail-0.2.0.dist-info/METADATA,sha256=7NqzFLPAM-6KFKU6Dk_bVF74t8FgxfFufzjpFaPJeRE,2545
+grib2sail-0.2.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+grib2sail-0.2.0.dist-info/top_level.txt,sha256=ubF1tLZ8ZWARUC9s6_0fRsuUKbVqOZmrp8UEGEBm8aE,10
+grib2sail-0.2.0.dist-info/RECORD,,

grib2sail-0.1.0.dist-info/RECORD

@@ -1,13 +0,0 @@
-grib2sail/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-grib2sail/__main__.py,sha256=U5PTpeJRtXGpcO8YPUdCD2hORF8T68IM6JW1-dDyYqE,37
-grib2sail/_version.py,sha256=5jwwVncvCiTnhOedfkzzxmxsggwmTBORdFL_4wq0ZeY,704
-grib2sail/cli.py,sha256=3IcGISWNF9XpDhSdSqb6z5bnc2YFmKlQKtTPH0M7hzI,2247
-grib2sail/downloader.py,sha256=9dyBk02ZA1XgB0DXkUtEmo1fMNosPR-B7fs4wPbLkQA,3581
-grib2sail/logger.py,sha256=_erYrZlKPjhDu-reFkyylodFgSVtOcXJg5F1drYeQNY,456
-grib2sail/token.py,sha256=ciyJNhJuXCHlPEFOIJfqpE0XNJr3FwGbdJmhagwzy04,1346
-grib2sail/variables.py,sha256=JInLyk0ShWBffNjokDW7u2RAyRSUj9KZxRiQXVppwU4,1266
-grib2sail-0.1.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-grib2sail-0.1.0.dist-info/METADATA,sha256=7Oa_OtDxJgJyI0Q7e1TmaYGtVrVaF64gFKj1dXj8wRs,2527
-grib2sail-0.1.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-grib2sail-0.1.0.dist-info/top_level.txt,sha256=ubF1tLZ8ZWARUC9s6_0fRsuUKbVqOZmrp8UEGEBm8aE,10
-grib2sail-0.1.0.dist-info/RECORD,,