libmozdata 0.2.15__tar.gz → 0.2.16__tar.gz

{libmozdata-0.2.15/libmozdata.egg-info → libmozdata-0.2.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: libmozdata
-Version: 0.2.15
+Version: 0.2.16
 Summary: Library to access and aggregate several Mozilla data sources.
 Home-page: https://github.com/mozilla/libmozdata
 Author: Mozilla Release Management

libmozdata-0.2.16/VERSION

@@ -0,0 +1 @@
+0.2.16

{libmozdata-0.2.15 → libmozdata-0.2.16}/libmozdata/lando.py

@@ -6,9 +6,8 @@
 from collections import namedtuple
 from urllib.parse import urljoin
 
-import requests
-
 from . import config
+from .utils import get_session
 
 
 class LandoWarnings(object):
@@ -28,7 +27,7 @@ class LandoWarnings(object):
         for warning in warnings:
             warning_id = warning["id"]
 
-            response = requests.delete(
+            response = get_session("lando").delete(
                 f"{self.api_url}{warning_id}",
                 headers={
                     "X-Phabricator-API-Key": self.api_key,
@@ -45,7 +44,7 @@ class LandoWarnings(object):
         """
         Adds a warning to Lando
         """
-        response = requests.post(
+        response = get_session("lando").post(
             self.api_url,
             json={
                 "revision_id": revision_id,
@@ -67,7 +66,7 @@ class LandoWarnings(object):
         """
         Gets a list of warnings
         """
-        response = requests.get(
+        response = get_session("lando").get(
             self.api_url,
             params={
                 "revision_id": revision_id,
@@ -122,7 +121,7 @@ class LandoCommitMapAPI:
         with both full hashes
         """
         url = urljoin(self.api_url, f"{method}/{repository}/{revision}")
-        resp = requests.get(
+        resp = get_session("lando").get(
             url,
             headers={
                 "User-Agent": self.USER_AGENT,

{libmozdata-0.2.15 → libmozdata-0.2.16}/libmozdata/phabricator.py

@@ -7,15 +7,13 @@ import collections
 import enum
 import json
 import logging
-import math
-import time
 from functools import cached_property
 from urllib.parse import urlencode, urlparse
 
 import hglib
-import requests
 
 from . import config
+from .utils import get_session
 
 HGMO_JSON_REV_URL_TEMPLATE = "https://hg.mozilla.org/mozilla-central/json-rev/{}"
 MOZILLA_PHABRICATOR_PROD = "https://phabricator.services.mozilla.com/api/"
@@ -210,18 +208,12 @@ class PhabricatorAPI(object):
     Phabricator Rest API client
     """
 
-    def __init__(self, api_key, url=MOZILLA_PHABRICATOR_PROD, max_retries=5):
+    def __init__(self, api_key, url=MOZILLA_PHABRICATOR_PROD):
         self.USER_AGENT = config.get("User-Agent", "name", required=True)
         self.api_key = api_key
         self.url = url
         assert self.url.endswith("/api/"), "Phabricator API must end with /api/"
 
-        # Number of API calls retries on 50x before raising an error
-        self.max_retries = max_retries
-        assert (
-            isinstance(self.max_retries, int) and self.max_retries > 0
-        ), "Invalid max_retries parameter"
-
     @cached_property
     def user(self):
         """
@@ -669,23 +661,11 @@ class PhabricatorAPI(object):
         # Add api token to payload
         payload["__conduit__"] = {"token": self.api_key}
 
-        # Run POST request on api, retrying on 50x errors
-        for nb_try in range(1, self.max_retries + 1):
-            response = requests.post(
-                self.url + path,
-                headers=self.get_header(),
-                data=urlencode({"params": json.dumps(payload), "output": "json"}),
-            )
-            if response.status_code >= 500:
-                # Retry after a while on 50x using exponential backoff
-                backoff = math.exp(nb_try)
-                logger.info(
-                    f"Phabricator failed with status code {response.status_code}, retrying in {backoff:.1f} seconds..."
-                )
-                time.sleep(backoff)
-            else:
-                # Stop retrying on the first successful try or other HTTP errors
-                break
+        response = get_session("phabricator").post(
+            self.url + path,
+            headers=self.get_header(),
+            data=urlencode({"params": json.dumps(payload), "output": "json"}),
+        )
 
         # Raise immediate 40x errors or out-of-retries 50x
         response.raise_for_status()

{libmozdata-0.2.15 → libmozdata-0.2.16}/libmozdata/utils.py

@@ -8,17 +8,61 @@ import operator
 import os.path
 import random
 from datetime import date, datetime, timedelta, timezone
+from functools import cache
 from itertools import count
 
 import dateutil.parser
 import pytz
+import requests
 import six
 from dateutil.relativedelta import relativedelta
 from requests.utils import quote
+from urllib3.util.retry import Retry
 
 __pacific = pytz.timezone("US/Pacific")
 
 
+@cache
+def get_session(name: str) -> requests.Session:
+    """Get a `requests` session with retries enabled on connection errors and
+    50x HTTP error codes, using exponential backoff.
+
+    Sessions are cached by name, so calling this function multiple times with
+    the same name will return the same session.
+
+    Args:
+        name (str): an arbitrary name identifying the caller, used to cache
+            the session
+
+    Returns:
+        requests.Session: a session with retries enabled
+    """
+    session = requests.Session()
+
+    retry = Retry(
+        total=8,
+        backoff_factor=1,
+        status_forcelist=[429, 500, 502, 503, 504],
+        # Also retry non-idempotent methods (e.g. POST), since urllib3
+        # excludes them by default.
+        allowed_methods=None,
+        # Return the last response instead of raising, once retries are
+        # exhausted, so callers can keep inspecting status_code/text.
+        raise_on_status=False,
+    )
+
+    # Default HTTPAdapter uses 10 connections. Mount custom adapter to increase
+    # that limit. Connections are established as needed, so using a large value
+    # should not negatively impact performance.
+    http_adapter = requests.adapters.HTTPAdapter(
+        pool_connections=50, pool_maxsize=50, max_retries=retry
+    )
+    session.mount("https://", http_adapter)
+    session.mount("http://", http_adapter)
+
+    return session
+
+
 def get_best(stats):
     """Get the key which has the higher value
 

{libmozdata-0.2.15 → libmozdata-0.2.16/libmozdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: libmozdata
-Version: 0.2.15
+Version: 0.2.16
 Summary: Library to access and aggregate several Mozilla data sources.
 Home-page: https://github.com/mozilla/libmozdata
 Author: Mozilla Release Management

{libmozdata-0.2.15 → libmozdata-0.2.16}/libmozdata.egg-info/requires.txt

@@ -6,4 +6,5 @@ requests-futures>=0.9.8
 requests[security]>=2.7.0
 setuptools>=83.0.0
 six>=1.17.0
+urllib3>=1.26.0
 whatthepatch>=0.0.4

{libmozdata-0.2.15 → libmozdata-0.2.16}/requirements.txt

@@ -6,4 +6,5 @@ requests-futures>=0.9.8
 requests[security]>=2.7.0
 setuptools>=83.0.0
 six>=1.17.0
+urllib3>=1.26.0
 whatthepatch>=0.0.4

libmozdata-0.2.15/VERSION

@@ -1 +0,0 @@
-0.2.15