libmozdata 0.2.15__tar.gz → 0.2.18__tar.gz

{libmozdata-0.2.15/libmozdata.egg-info → libmozdata-0.2.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: libmozdata
-Version: 0.2.15
+Version: 0.2.18
 Summary: Library to access and aggregate several Mozilla data sources.
 Home-page: https://github.com/mozilla/libmozdata
 Author: Mozilla Release Management

libmozdata-0.2.18/VERSION

@@ -0,0 +1 @@
+0.2.18

{libmozdata-0.2.15 → libmozdata-0.2.18}/libmozdata/HGFileInfo.py

@@ -5,8 +5,6 @@
 import logging
 import re
 
-import six
-
 from . import hgmozilla
 from .connection import Query
 
@@ -32,7 +30,7 @@ class HGFileInfo(object):
         self.node = node
         self.date_type = "date" if date_type == "creation" else "pushdate"
         self.data = {}
-        self.paths = [paths] if isinstance(paths, six.string_types) else paths
+        self.paths = [paths] if isinstance(paths, str) else paths
         for p in self.paths:
             self.data[p] = []
         self.bug_pattern = re.compile(r"[\t ]*[Bb][Uu][Gg][\t ]*([0-9]+)")

{libmozdata-0.2.15 → libmozdata-0.2.18}/libmozdata/bugzilla.py

@@ -6,7 +6,6 @@ import functools
 import re
 
 import requests
-import six
 from requests import HTTPError
 
 import libmozdata.versions
@@ -75,7 +74,7 @@ class Bugzilla(BugzillaBase):
             super(Bugzilla, self).__init__(Bugzilla.URL, queries=queries, **kwargs)
         else:
             super(Bugzilla, self).__init__(Bugzilla.URL, **kwargs)
-            if isinstance(bugids, six.string_types) or isinstance(bugids, dict):
+            if isinstance(bugids, str) or isinstance(bugids, dict):
                 self.bugids = [bugids]
             elif isinstance(bugids, int):
                 self.bugids = [str(bugids)]
@@ -196,9 +195,9 @@ class Bugzilla(BugzillaBase):
 
         def __merge_fields(f1, f2):
             if f1:
-                f1 = {f1} if isinstance(f1, six.string_types) else set(f1)
+                f1 = {f1} if isinstance(f1, str) else set(f1)
                 if f2:
-                    f2 = {f2} if isinstance(f2, six.string_types) else set(f2)
+                    f2 = {f2} if isinstance(f2, str) else set(f2)
                     return list(f1.union(f2))
                 else:
                     return f1
@@ -257,7 +256,7 @@ class Bugzilla(BugzillaBase):
 
     @staticmethod
     def get_links(bugids):
-        if isinstance(bugids, six.string_types) or isinstance(bugids, int):
+        if isinstance(bugids, str) or isinstance(bugids, int):
             return "https://bugzilla.mozilla.org/" + str(bugids)
         else:
             return ["https://bugzilla.mozilla.org/" + str(bugid) for bugid in bugids]
@@ -594,7 +593,7 @@ class Bugzilla(BugzillaBase):
         header = self.get_header()
         specials = {"count_only", "limit", "order", "offset"}
         for query in self.bugids:
-            if isinstance(query, six.string_types):
+            if isinstance(query, str):
                 url = Bugzilla.API_URL + "?" + query
                 self.bugs_results.append(
                     self.session.get(
@@ -730,7 +729,7 @@ class Bugzilla(BugzillaBase):
                 bugs = json["bugs"]
                 if bugs:
                     for key in bugs.keys():
-                        if isinstance(key, six.string_types) and key.isdigit():
+                        if isinstance(key, str) and key.isdigit():
                             comments = bugs[key]
                             self.commenthandler.handle(comments, key)
         elif self.RAISE_ERROR:
@@ -775,7 +774,7 @@ class Bugzilla(BugzillaBase):
                 bugs = json["bugs"]
                 if bugs:
                     for key in bugs.keys():
-                        if isinstance(key, six.string_types) and key.isdigit():
+                        if isinstance(key, str) and key.isdigit():
                             attachments = bugs[key]
                             self.attachmenthandler.handle(attachments, key)
         elif self.RAISE_ERROR:
@@ -861,7 +860,7 @@ class BugzillaUser(BugzillaBase):
         self.fault_user_handler = Handler.get(fault_user_handler, user_data)
 
         if user_names is not None:
-            if isinstance(user_names, six.string_types) or isinstance(user_names, int):
+            if isinstance(user_names, str) or isinstance(user_names, int):
                 user_names = [user_names]
 
             params = [
@@ -870,8 +869,7 @@ class BugzillaUser(BugzillaBase):
                     "names": [
                         user_name
                         for user_name in user_names
-                        if isinstance(user_name, six.string_types)
-                        and not user_name.isdigit()
+                        if isinstance(user_name, str) and not user_name.isdigit()
                     ],
                     "ids": [
                         str(user_id)
@@ -891,7 +889,7 @@ class BugzillaUser(BugzillaBase):
                 **kwargs,
             )
         elif search_strings is not None:
-            if isinstance(search_strings, six.string_types):
+            if isinstance(search_strings, str):
                 search_strings = [search_strings]
 
             queries = []
@@ -952,9 +950,9 @@ class BugzillaProduct(BugzillaBase):
 
             if include_fields:
                 if "include_fields" in params:
-                    if isinstance(include_fields, six.string_types):
+                    if isinstance(include_fields, str):
                         include_fields = [include_fields]
-                    if isinstance(params.get("include_fields"), six.string_types):
+                    if isinstance(params.get("include_fields"), str):
                         params["include_fields"] = [params["include_fields"]]
 
                     params["include_fields"] = list(
@@ -965,9 +963,9 @@ class BugzillaProduct(BugzillaBase):
 
             if product_types:
                 if "type" in params:
-                    if isinstance(product_types, six.string_types):
+                    if isinstance(product_types, str):
                         product_types = [product_types]
-                    if isinstance(params.get("type"), six.string_types):
+                    if isinstance(params.get("type"), str):
                         params["type"] = [params["type"]]
 
                     params["type"] = list(set(params["type"]).union(product_types))
@@ -975,9 +973,7 @@ class BugzillaProduct(BugzillaBase):
                     params["type"] = product_types
 
         elif product_names is not None:
-            if isinstance(product_names, six.string_types) or isinstance(
-                product_names, int
-            ):
+            if isinstance(product_names, str) or isinstance(product_names, int):
                 product_names = [product_names]
 
             params = {
@@ -986,8 +982,7 @@ class BugzillaProduct(BugzillaBase):
                 "names": [
                     product_name
                     for product_name in product_names
-                    if isinstance(product_name, six.string_types)
-                    and not product_name.isdigit()
+                    if isinstance(product_name, str) and not product_name.isdigit()
                 ],
                 "ids": [
                     str(product_id)

{libmozdata-0.2.15 → libmozdata-0.2.18}/libmozdata/connection.py

@@ -61,7 +61,8 @@ class Connection(object):
     )
 
     # Error 429 is for 'Too many requests' => we retry
-    STATUS_FORCELIST = [429]
+    # Also retry on server errors.
+    STATUS_FORCELIST = [429, 500, 502, 503, 504]
 
     def __init__(self, base_url, queries=None, **kwargs):
         """Constructor

{libmozdata-0.2.15 → libmozdata-0.2.18}/libmozdata/hgmozilla.py

@@ -2,8 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import six
-
 from . import config
 from .connection import Connection, Query
 
@@ -244,7 +242,7 @@ class FileInfo(Mercurial):
 
         __base = {"node": node, "file": None}
 
-        if isinstance(paths, six.string_types):
+        if isinstance(paths, str):
             __base["file"] = paths
             _dict = {}
             data[paths] = _dict
@@ -334,7 +332,7 @@ class Annotate(Mercurial):
 
         __base = {"node": node, "file": None}
 
-        if isinstance(paths, six.string_types):
+        if isinstance(paths, str):
             __base["file"] = paths
             _dict = {}
             data[paths] = _dict

{libmozdata-0.2.15 → libmozdata-0.2.18}/libmozdata/lando.py

@@ -6,9 +6,8 @@
 from collections import namedtuple
 from urllib.parse import urljoin
 
-import requests
-
 from . import config
+from .utils import get_session
 
 
 class LandoWarnings(object):
@@ -28,7 +27,7 @@ class LandoWarnings(object):
         for warning in warnings:
             warning_id = warning["id"]
 
-            response = requests.delete(
+            response = get_session("lando").delete(
                 f"{self.api_url}{warning_id}",
                 headers={
                     "X-Phabricator-API-Key": self.api_key,
@@ -45,7 +44,7 @@ class LandoWarnings(object):
         """
         Adds a warning to Lando
         """
-        response = requests.post(
+        response = get_session("lando").post(
             self.api_url,
             json={
                 "revision_id": revision_id,
@@ -67,7 +66,7 @@ class LandoWarnings(object):
         """
         Gets a list of warnings
         """
-        response = requests.get(
+        response = get_session("lando").get(
             self.api_url,
             params={
                 "revision_id": revision_id,
@@ -122,7 +121,7 @@ class LandoCommitMapAPI:
         with both full hashes
         """
         url = urljoin(self.api_url, f"{method}/{repository}/{revision}")
-        resp = requests.get(
+        resp = get_session("lando").get(
             url,
             headers={
                 "User-Agent": self.USER_AGENT,

{libmozdata-0.2.15 → libmozdata-0.2.18}/libmozdata/phabricator.py

@@ -7,15 +7,13 @@ import collections
 import enum
 import json
 import logging
-import math
-import time
 from functools import cached_property
 from urllib.parse import urlencode, urlparse
 
 import hglib
-import requests
 
 from . import config
+from .utils import get_session
 
 HGMO_JSON_REV_URL_TEMPLATE = "https://hg.mozilla.org/mozilla-central/json-rev/{}"
 MOZILLA_PHABRICATOR_PROD = "https://phabricator.services.mozilla.com/api/"
@@ -210,18 +208,12 @@ class PhabricatorAPI(object):
     Phabricator Rest API client
     """
 
-    def __init__(self, api_key, url=MOZILLA_PHABRICATOR_PROD, max_retries=5):
+    def __init__(self, api_key, url=MOZILLA_PHABRICATOR_PROD):
         self.USER_AGENT = config.get("User-Agent", "name", required=True)
         self.api_key = api_key
         self.url = url
         assert self.url.endswith("/api/"), "Phabricator API must end with /api/"
 
-        # Number of API calls retries on 50x before raising an error
-        self.max_retries = max_retries
-        assert (
-            isinstance(self.max_retries, int) and self.max_retries > 0
-        ), "Invalid max_retries parameter"
-
     @cached_property
     def user(self):
         """
@@ -669,23 +661,11 @@ class PhabricatorAPI(object):
         # Add api token to payload
         payload["__conduit__"] = {"token": self.api_key}
 
-        # Run POST request on api, retrying on 50x errors
-        for nb_try in range(1, self.max_retries + 1):
-            response = requests.post(
-                self.url + path,
-                headers=self.get_header(),
-                data=urlencode({"params": json.dumps(payload), "output": "json"}),
-            )
-            if response.status_code >= 500:
-                # Retry after a while on 50x using exponential backoff
-                backoff = math.exp(nb_try)
-                logger.info(
-                    f"Phabricator failed with status code {response.status_code}, retrying in {backoff:.1f} seconds..."
-                )
-                time.sleep(backoff)
-            else:
-                # Stop retrying on the first successful try or other HTTP errors
-                break
+        response = get_session("phabricator").post(
+            self.url + path,
+            headers=self.get_header(),
+            data=urlencode({"params": json.dumps(payload), "output": "json"}),
+        )
 
         # Raise immediate 40x errors or out-of-retries 50x
         response.raise_for_status()

{libmozdata-0.2.15 → libmozdata-0.2.18}/libmozdata/redash.py

@@ -6,8 +6,6 @@ import functools
 import re
 from datetime import timedelta
 
-import six
-
 from . import config, utils
 from .connection import Connection, Query
 
@@ -93,7 +91,7 @@ class Redash(Connection):
             dict: containing result in json for each query
         """
         data = {}
-        if isinstance(query_ids, six.string_types):
+        if isinstance(query_ids, str):
             url = Redash.API_URL + "/" + query_ids + "/results.json"
             Redash(
                 Query(

{libmozdata-0.2.15 → libmozdata-0.2.18}/libmozdata/socorro.py

@@ -2,8 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import six
-
 from . import config, utils
 from .connection import Connection, Query
 
@@ -85,7 +83,7 @@ class SuperSearch(Socorro):
                     "url" in k
                     or "email" in k
                     or (
-                        (isinstance(v, list) or isinstance(v, six.string_types))
+                        (isinstance(v, list) or isinstance(v, str))
                         and ("url" in v or "email" in v)
                     )
                 ):
@@ -175,7 +173,7 @@ class ProcessedCrash(Socorro):
         data = {}
         __base = {"crash_id": None, "datatype": "processed"}
 
-        if isinstance(crashids, six.string_types):
+        if isinstance(crashids, str):
             __base["crash_id"] = crashids
             _dict = {}
             data[crashids] = _dict
@@ -243,7 +241,7 @@ class Bugs(Socorro):
                     if signature in data:
                         data[signature].add(hit["id"])
 
-        if isinstance(signatures, six.string_types):
+        if isinstance(signatures, str):
             data = {signatures: set()}
             Bugs(
                 params={"signatures": signatures},

{libmozdata-0.2.15 → libmozdata-0.2.18}/libmozdata/utils.py

@@ -8,17 +8,60 @@ import operator
 import os.path
 import random
 from datetime import date, datetime, timedelta, timezone
+from functools import cache
 from itertools import count
 
 import dateutil.parser
 import pytz
-import six
+import requests
 from dateutil.relativedelta import relativedelta
 from requests.utils import quote
+from urllib3.util.retry import Retry
 
 __pacific = pytz.timezone("US/Pacific")
 
 
+@cache
+def get_session(name: str) -> requests.Session:
+    """Get a `requests` session with retries enabled on connection errors and
+    50x HTTP error codes, using exponential backoff.
+
+    Sessions are cached by name, so calling this function multiple times with
+    the same name will return the same session.
+
+    Args:
+        name (str): an arbitrary name identifying the caller, used to cache
+            the session
+
+    Returns:
+        requests.Session: a session with retries enabled
+    """
+    session = requests.Session()
+
+    retry = Retry(
+        total=8,
+        backoff_factor=1,
+        status_forcelist=[429, 500, 502, 503, 504],
+        # Also retry non-idempotent methods (e.g. POST), since urllib3
+        # excludes them by default.
+        allowed_methods=None,
+        # Return the last response instead of raising, once retries are
+        # exhausted, so callers can keep inspecting status_code/text.
+        raise_on_status=False,
+    )
+
+    # Default HTTPAdapter uses 10 connections. Mount custom adapter to increase
+    # that limit. Connections are established as needed, so using a large value
+    # should not negatively impact performance.
+    http_adapter = requests.adapters.HTTPAdapter(
+        pool_connections=50, pool_maxsize=50, max_retries=retry
+    )
+    session.mount("https://", http_adapter)
+    session.mount("http://", http_adapter)
+
+    return session
+
+
 def get_best(stats):
     """Get the key which has the higher value
 
@@ -43,7 +86,7 @@ def get_timestamp(dt):
     Returns:
         int: the corresponding timestamp
     """
-    if isinstance(dt, six.string_types):
+    if isinstance(dt, str):
         dt = datetime.now(timezone.utc) if dt == "now" else get_date_ymd(dt)
     return int(calendar.timegm(dt.timetuple()))
 
@@ -118,7 +161,7 @@ def get_date(_date, delta=None):
     Returns:
         str: the date as a string 'Year-month-day'
     """
-    if isinstance(_date, six.string_types):
+    if isinstance(_date, str):
         _date = get_date_ymd(_date)
     if delta:
         _date -= timedelta(delta)
@@ -307,16 +350,16 @@ def get_channels():
 
 
 def get_str_list(x):
-    if isinstance(x, six.string_types):
+    if isinstance(x, str):
         return [x]
-    if isinstance(x, six.integer_types):
+    if isinstance(x, int):
         return [str[x]]
     if isinstance(x, list) or isinstance(x, set):
-        return [str(y) if isinstance(y, six.integer_types) else y for y in x]
+        return [str(y) if isinstance(y, int) else y for y in x]
 
 
 def get_x_fwed_for_str(s):
-    if isinstance(s, six.string_types):
+    if isinstance(s, str):
         return ", ".join(map(lambda x: x.strip(" \t"), s.split(",")))
     else:
         return ", ".join(map(lambda x: x.strip(" \t"), s))

{libmozdata-0.2.15 → libmozdata-0.2.18/libmozdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: libmozdata
-Version: 0.2.15
+Version: 0.2.18
 Summary: Library to access and aggregate several Mozilla data sources.
 Home-page: https://github.com/mozilla/libmozdata
 Author: Mozilla Release Management

{libmozdata-0.2.15 → libmozdata-0.2.18}/libmozdata.egg-info/requires.txt

@@ -5,5 +5,5 @@ pytz>=2022
 requests-futures>=0.9.8
 requests[security]>=2.7.0
 setuptools>=83.0.0
-six>=1.17.0
+urllib3>=1.26.0
 whatthepatch>=0.0.4

{libmozdata-0.2.15 → libmozdata-0.2.18}/requirements.txt

@@ -5,5 +5,5 @@ pytz>=2022
 requests-futures>=0.9.8
 requests[security]>=2.7.0
 setuptools>=83.0.0
-six>=1.17.0
+urllib3>=1.26.0
 whatthepatch>=0.0.4

libmozdata-0.2.15/VERSION

@@ -1 +0,0 @@
-0.2.15