libmozdata 0.2.3__tar.gz → 0.2.6__tar.gz

{libmozdata-0.2.3/libmozdata.egg-info → libmozdata-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: libmozdata
-Version: 0.2.3
+Version: 0.2.6
 Summary: Library to access and aggregate several Mozilla data sources.
 Home-page: https://github.com/mozilla/libmozdata
 Author: Mozilla Release Management

libmozdata-0.2.6/VERSION

@@ -0,0 +1 @@
+0.2.6

{libmozdata-0.2.3 → libmozdata-0.2.6}/libmozdata/bugzilla.py

@@ -16,15 +16,23 @@ from .connection import Connection, Query
 from .handler import Handler
 
 
-class Bugzilla(Connection):
-    """Connection to bugzilla.mozilla.org"""
-
+class BugzillaBase(Connection):
     URL = config.get("Bugzilla", "URL", "https://bugzilla.mozilla.org")
     # URL = config.get('Allizgub', 'URL', 'https://bugzilla-dev.allizom.org')
-    API_URL = URL + "/rest/bug"
-    ATTACHMENT_API_URL = API_URL + "/attachment"
     TOKEN = config.get("Bugzilla", "token", "")
     # TOKEN = config.get('Allizgub', 'token', '')
+
+    def get_header(self):
+        header = super(BugzillaBase, self).get_header()
+        header["X-Bugzilla-API-Key"] = self.get_apikey()
+        return header
+
+
+class Bugzilla(BugzillaBase):
+    """Connection to bugzilla.mozilla.org"""
+
+    API_URL = BugzillaBase.URL + "/rest/bug"
+    ATTACHMENT_API_URL = API_URL + "/attachment"
     BUGZILLA_CHUNK_SIZE = 100
 
     def __init__(
@@ -94,11 +102,6 @@ class Bugzilla(Connection):
             self.got_data = False
             self.no_private_bugids = None
 
-    def get_header(self):
-        header = super(Bugzilla, self).get_header()
-        header["X-Bugzilla-API-Key"] = self.get_apikey()
-        return header
-
     def put(self, data, attachment=False, retry_on_failure=False):
         """Put some data in bugs
 
@@ -816,12 +819,10 @@ class Bugzilla(Connection):
             )
 
 
-class BugzillaUser(Connection):
+class BugzillaUser(BugzillaBase):
     """Connection to bugzilla.mozilla.org"""
 
-    URL = config.get("Bugzilla", "URL", "https://bugzilla.mozilla.org")
-    API_URL = URL + "/rest/user"
-    TOKEN = config.get("Bugzilla", "token", "")
+    API_URL = BugzillaBase.URL + "/rest/user"
 
     def __init__(
         self,
@@ -892,11 +893,6 @@ class BugzillaUser(Connection):
 
             super(BugzillaUser, self).__init__(BugzillaUser.URL, queries, **kwargs)
 
-    def get_header(self):
-        header = super(BugzillaUser, self).get_header()
-        header["X-Bugzilla-API-Key"] = self.get_apikey()
-        return header
-
     def __users_cb(self, res):
         if self.user_handler.isactive():
             for user in res["users"]:
@@ -907,16 +903,14 @@ class BugzillaUser(Connection):
                 self.fault_user_handler.handle(user)
 
 
-class BugzillaProduct(Connection):
+class BugzillaProduct(BugzillaBase):
     """
     Connection to bugzilla.mozilla.org
 
     API docs: https://bugzilla.readthedocs.io/en/latest/api/core/v1/product.html
     """
 
-    URL = config.get("Bugzilla", "URL", "https://bugzilla.mozilla.org")
-    API_URL = URL + "/rest/product"
-    TOKEN = config.get("Bugzilla", "token", "")
+    API_URL = BugzillaBase.URL + "/rest/product"
 
     def __init__(
         self,
@@ -1007,11 +1001,6 @@ class BugzillaProduct(Connection):
             **kwargs,
         )
 
-    def get_header(self):
-        header = super(BugzillaProduct, self).get_header()
-        header["X-Bugzilla-API-Key"] = self.get_apikey()
-        return header
-
     def __products_cb(self, res):
         if not self.product_handler.isactive():
             return
@@ -1020,14 +1009,12 @@ class BugzillaProduct(Connection):
             self.product_handler.handle(product)
 
 
-class BugzillaShorten(Connection):
+class BugzillaShorten(BugzillaBase):
     """
     Connection to bugzilla.mozilla.org
     """
 
-    URL = config.get("Bugzilla", "URL", "https://bugzilla.mozilla.org")
-    API_URL = URL + "/rest/bitly/shorten"
-    TOKEN = config.get("Bugzilla", "token", "")
+    API_URL = BugzillaBase.URL + "/rest/bitly/shorten"
 
     def __init__(self, url, url_data=None, url_handler=None, **kwargs):
         """Constructor
@@ -1049,11 +1036,6 @@ class BugzillaShorten(Connection):
             **kwargs,
         )
 
-    def get_header(self):
-        header = super(BugzillaShorten, self).get_header()
-        header["X-Bugzilla-API-Key"] = self.get_apikey()
-        return header
-
     def __urls_cb(self, res):
         if not self.url_handler.isactive():
             return
@@ -1069,14 +1051,12 @@ class BugzillaShorten(Connection):
         self.url_handler.handle(res["url"])
 
 
-class BugzillaComponent(Connection):
+class BugzillaComponent(BugzillaBase):
     """
     Connection to bugzilla.mozilla.org
     """
 
-    URL = config.get("Bugzilla", "URL", "https://bugzilla.mozilla.org")
-    API_URL = URL + "/rest/component"
-    TOKEN = config.get("Bugzilla", "token", "")
+    API_URL = BugzillaBase.URL + "/rest/component"
 
     def __init__(
         self, product, component, component_data=None, component_handler=None, **kwargs
@@ -1105,11 +1085,6 @@ class BugzillaComponent(Connection):
             **kwargs,
         )
 
-    def get_header(self):
-        header = super(BugzillaComponent, self).get_header()
-        header["X-Bugzilla-API-Key"] = self.get_apikey()
-        return header
-
     def __components_cb(self, res):
         if not self.component_handler.isactive():
             return
@@ -1135,3 +1110,86 @@ class BugzillaComponent(Connection):
         response.raise_for_status()
 
         return response.json()
+
+
+class BugFields(BugzillaBase):
+    """Fetching bug fields information from Bugzilla
+
+    Docs: https://bmo.readthedocs.io/en/latest/api/core/v1/field.html#fields
+    """
+
+    API_URL = BugzillaBase.URL + "/rest/field/bug"
+
+    def __init__(
+        self, field=None, handler=None, handlerdata=None, queries=None, **kwargs
+    ):
+        """Constructor
+
+        Args:
+            field (Optional[str]): the name of the field, if empty will fetch all fields
+            handler (Optional[function]): the handler to use with each retrieved field
+            handlerdata (Optional): the data to use with the field handler
+            queries (Optional[Query]): the queries to use
+        """
+        if not queries:
+            endpoint_url = f"{self.API_URL}/{field}" if field else self.API_URL
+            queries = Query(endpoint_url, None, handler, handlerdata)
+
+        super(BugFields, self).__init__(self.URL, queries, **kwargs)
+
+    @classmethod
+    def fetch_all_fields_info(cls):
+        """Get information about all fields
+
+        Returns:
+            list: the information about the fields
+        """
+
+        def handler(resp, data):
+            data.update(resp)
+
+        data = {}
+        cls(
+            handler=handler,
+            handlerdata=data,
+        ).wait()
+
+        return data["fields"]
+
+    @classmethod
+    def fetch_field_info(cls, field):
+        """Get information about a field
+
+        Args:
+            field (str): the name of the field
+
+        Returns:
+            dict: the information about the field
+        """
+
+        def handler(resp, data):
+            data.update(resp)
+
+        data = {}
+        cls(
+            field=field,
+            handler=handler,
+            handlerdata=data,
+        ).wait()
+
+        field_info = data["fields"][0]
+        assert field_info["name"] == field
+
+        return field_info
+
+    @classmethod
+    def fetch_field_values(cls, field):
+        """Get the legal values of a field
+
+        Args:
+            field (str): the name of the field
+
+        Returns:
+            List[str]: the legal values of the field
+        """
+        return [value["name"] for value in cls.fetch_field_info(field)["values"]]

libmozdata-0.2.6/libmozdata/clouseau.py

@@ -0,0 +1,76 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+from libmozdata.utils import batched
+
+from . import config
+from .connection import Connection, Query
+
+
+class CrashClouseau(Connection):
+    URL = config.get("Clouseau", "URL", "https://clouseau.moz.tools")
+    API_URL = URL + "/api"
+
+
+class Reports(CrashClouseau):
+    API_URL = CrashClouseau.API_URL + "/reports"
+
+    def __init__(
+        self, params=None, handler=None, handlerdata=None, queries=None, **kwargs
+    ):
+        """Constructor
+
+        Args:
+            params (Optional[dict]): the params for the query
+            handler (Optional[function]): handler to use with the result of the query
+            handlerdata (Optional): data used in second argument of the handler
+            queries (Optional[List[Query]]): queries to execute
+        """
+        if queries is not None:
+            super().__init__(self.URL, queries, **kwargs)
+        else:
+            super().__init__(
+                self.URL, Query(self.API_URL, params, handler, handlerdata), **kwargs
+            )
+
+    @staticmethod
+    def _default_handler(res, data):
+        for report in res:
+            signature = report["signature"]
+            if signature not in data:
+                data[signature] = [report]
+            else:
+                data[signature].append(report)
+
+    @classmethod
+    def get_by_signatures(cls, signatures, product=None, channel=None):
+        """Get reports by signatures
+
+        Args:
+            signatures: signatures to get their reports.
+            product: filter out reports that are not from this product.
+            channel: filter out reports that are not from this release channel.
+
+        Returns:
+            dict: the reports by signatures
+        """
+        data = {}
+        requests = [
+            cls(
+                params={
+                    "signatures": signatures_batch,
+                    "product": product,
+                    "channel": channel,
+                },
+                handler=cls._default_handler,
+                handlerdata=data,
+            )
+            for signatures_batch in batched(signatures, 20)
+        ]
+
+        for request in requests:
+            request.wait()
+
+        return data

{libmozdata-0.2.3 → libmozdata-0.2.6}/libmozdata/connection.py

@@ -48,6 +48,7 @@ class Query(object):
 class Connection(object):
     """Represents a connection to a server"""
 
+    VERIFY_SSL = True
     TIMEOUT = 30
     MAX_RETRIES = 256
     MAX_WORKERS = multiprocessing.cpu_count()
@@ -190,7 +191,7 @@ class Connection(object):
                                 params=query.params,
                                 headers=header,
                                 auth=auth,
-                                verify=True,
+                                verify=self.VERIFY_SSL,
                                 timeout=self.TIMEOUT,
                                 hooks={"response": cb},
                             )
@@ -203,7 +204,7 @@ class Connection(object):
                                     params=p,
                                     headers=header,
                                     auth=auth,
-                                    verify=True,
+                                    verify=self.VERIFY_SSL,
                                     timeout=self.TIMEOUT,
                                     hooks={"response": cb},
                                 )
@@ -214,7 +215,7 @@ class Connection(object):
                             query.url,
                             headers=header,
                             auth=auth,
-                            verify=True,
+                            verify=self.VERIFY_SSL,
                             timeout=self.TIMEOUT,
                             hooks={"response": cb},
                         )

{libmozdata-0.2.3 → libmozdata-0.2.6}/libmozdata/phabricator.py

@@ -7,6 +7,7 @@ import collections
 import enum
 import json
 import logging
+from functools import cached_property
 from urllib.parse import urlencode, urlparse
 
 import hglib
@@ -213,9 +214,12 @@ class PhabricatorAPI(object):
         self.url = url
         assert self.url.endswith("/api/"), "Phabricator API must end with /api/"
 
-        # Test authentication
-        self.user = self.request("user.whoami")
-        logger.info("Authenticated on {} as {}".format(self.url, self.user["realName"]))
+    @cached_property
+    def user(self):
+        """
+        Return the authenticated user details
+        """
+        return self.request("user.whoami")
 
     @property
     def hostname(self):

{libmozdata-0.2.3 → libmozdata-0.2.6}/libmozdata/release_owners.py

@@ -48,7 +48,6 @@ def get_owners():
             "Secondary",
             "Engineering REO",
             "Release Duty",
-            "QA Owner",
             "Corresponding ESR",
             "Release Date",
         ] != table[0]:
@@ -58,7 +57,7 @@ def get_owners():
         for row in table[1:]:
             try:
                 # sometimes the date is 2019-XX-XX (when the date is not known)
-                release_date = utils.get_date_ymd(row[7])
+                release_date = utils.get_date_ymd(row[6])
             except (AssertionError, ValueError):
                 continue
 
@@ -69,8 +68,7 @@ def get_owners():
                     "secondary": row[2],
                     "engineering reo": row[3],
                     "release duty": _get_list_people(row[4]),
-                    "qa owner": row[5],
-                    "corresponding esr": row[6],
+                    "corresponding esr": row[5],
                     "release date": release_date,
                 }
             )

{libmozdata-0.2.3 → libmozdata-0.2.6}/libmozdata/socorro.py

@@ -30,10 +30,9 @@ class Socorro(Connection):
 
 
 class SuperSearch(Socorro):
-    """SuperSearch: https://crash-stats.mozilla.org/search/?product=&_dont_run=1"""
+    """SuperSearch: https://crash-stats.mozilla.org/api/#SuperSearch"""
 
     URL = Socorro.API_URL + "/SuperSearch/"
-    URL_UNREDACTED = Socorro.API_URL + "/SuperSearchUnredacted/"
     WEB_URL = Socorro.CRASH_STATS_URL + "/search/"
 
     def __init__(
@@ -50,35 +49,49 @@ class SuperSearch(Socorro):
         if queries is not None:
             super(SuperSearch, self).__init__(queries, **kwargs)
         else:
-            url = SuperSearch.URL
-            unredacted = False
-            if "_facets" in params:
-                facets = params["_facets"]
-                if "url" in facets or "email" in facets:
-                    url = SuperSearch.URL_UNREDACTED
-                    unredacted = True
-            if not unredacted and "_columns" in params:
-                columns = params["_columns"]
-                if "url" in columns or "email" in columns:
-                    url = SuperSearch.URL_UNREDACTED
-            if not unredacted:
-                for k, v in params.items():
-                    if (
-                        "url" in k
-                        or "email" in k
-                        or (
-                            (isinstance(v, list) or isinstance(v, six.string_types))
-                            and ("url" in v or "email" in v)
-                        )
-                    ):
-                        url = SuperSearch.URL_UNREDACTED
-                        unredacted = True
-                        break
-
+            if self.__has_deprecated_unredacted_params(params):
+                raise ValueError(
+                    "Requesting PII data using the `SuperSearch` class is not "
+                    "supported anymore. Please use `SuperSearchUnredacted` instead."
+                )
             super(SuperSearch, self).__init__(
-                Query(url, params, handler, handlerdata), **kwargs
+                Query(self.URL, params, handler, handlerdata), **kwargs
             )
 
+    def __has_deprecated_unredacted_params(self, params):
+        """Check if the params is requesting PII data that we used to
+        automatically support retrieving it by redirecting the request to the
+        unredacted endpoint (i.e., SuperSearchUnredacted).
+        """
+        if not isinstance(params, dict):
+            # This is a workaround to avoid crashing when the params values is a
+            # list of params. We could instead of this, check each item in the
+            # list, but it's not worth it since this method is just for backward
+            # compatibility.
+            return False
+
+        unredacted = False
+        if "_facets" in params:
+            facets = params["_facets"]
+            if "url" in facets or "email" in facets:
+                unredacted = True
+        if not unredacted and "_columns" in params:
+            columns = params["_columns"]
+            if "url" in columns or "email" in columns:
+                unredacted = True
+        if not unredacted:
+            for k, v in params.items():
+                if (
+                    "url" in k
+                    or "email" in k
+                    or (
+                        (isinstance(v, list) or isinstance(v, six.string_types))
+                        and ("url" in v or "email" in v)
+                    )
+                ):
+                    unredacted = True
+        return unredacted
+
     @staticmethod
     def get_link(params):
         return utils.get_url(SuperSearch.WEB_URL) + utils.get_params_for_url(params)
@@ -109,6 +122,12 @@ class SuperSearch(Socorro):
         return search_date
 
 
+class SuperSearchUnredacted(SuperSearch):
+    """SuperSearchUnredacted: https://crash-stats.mozilla.org/api/#SuperSearchUnredacted"""
+
+    URL = Socorro.API_URL + "/SuperSearchUnredacted/"
+
+
 class ProcessedCrash(Socorro):
     """ProcessedCrash: https://crash-stats.mozilla.org/api/#ProcessedCrash"""
 
@@ -244,3 +263,50 @@ class Bugs(Socorro):
             data[k] = list(v)
 
         return data
+
+
+class SignatureFirstDate(Socorro):
+    """SignatureFirstDate: https://crash-stats.mozilla.org/api/#SignatureFirstDate"""
+
+    URL = Socorro.API_URL + "/SignatureFirstDate/"
+
+    def __init__(
+        self, params=None, handler=None, handlerdata=None, queries=None, **kwargs
+    ):
+        """Constructor
+
+        Args:
+            params (Optional[dict]): the params for the query
+            handler (Optional[function]): handler to use with the result of the query
+            handlerdata (Optional): data used in second argument of the handler
+            queries (Optional[List[Query]]): queries to execute
+        """
+        if queries:
+            super(SignatureFirstDate, self).__init__(queries, **kwargs)
+        else:
+            super(SignatureFirstDate, self).__init__(
+                Query(SignatureFirstDate.URL, params, handler, handlerdata), **kwargs
+            )
+
+    @staticmethod
+    def get_signatures(signatures):
+        """Get the first date and build id for the specified signatures.
+
+        Args:
+            signatures (List[str]): signatures to check.
+
+        Returns:
+            dict: the first date for each signature.
+        """
+
+        def default_handler(json, data):
+            data.extend(json["hits"])
+
+        data = []
+        SignatureFirstDate(
+            params={"signatures": signatures},
+            handler=default_handler,
+            handlerdata=data,
+        ).wait()
+
+        return data

{libmozdata-0.2.3 → libmozdata-0.2.6}/libmozdata/utils.py

@@ -8,6 +8,7 @@ import operator
 import os.path
 import random
 from datetime import date, datetime, timedelta
+from itertools import count
 
 import dateutil.parser
 import pytz
@@ -381,3 +382,39 @@ def get_language(path):
     for lang, names in langs.items():
         if name in names or extension in names:
             return lang
+
+
+# This is roughly equivalent to the itertools.islice which was added in
+# Python 3.10.
+# Copied from https://docs.python.org/3/library/itertools.html#itertools.islice
+def islice(iterable, *args):
+    # islice('ABCDEFG', 2) → A B
+    # islice('ABCDEFG', 2, 4) → C D
+    # islice('ABCDEFG', 2, None) → C D E F G
+    # islice('ABCDEFG', 0, None, 2) → A C E G
+
+    s = slice(*args)
+    start = 0 if s.start is None else s.start
+    stop = s.stop
+    step = 1 if s.step is None else s.step
+    if start < 0 or (stop is not None and stop < 0) or step <= 0:
+        raise ValueError
+
+    indices = count() if stop is None else range(max(start, stop))
+    next_i = start
+    for i, element in zip(indices, iterable):
+        if i == next_i:
+            yield element
+            next_i += step
+
+
+# This is roughly equivalent to the itertools.batched which was added in
+# Python 3.12.
+# Copied from https://docs.python.org/3/library/itertools.html#itertools.batched
+def batched(iterable, n):
+    # batched('ABCDEFG', 3) → ABC DEF G
+    if n < 1:
+        raise ValueError("n must be at least one")
+    iterator = iter(iterable)
+    while batch := tuple(islice(iterator, n)):
+        yield batch

{libmozdata-0.2.3 → libmozdata-0.2.6/libmozdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: libmozdata
-Version: 0.2.3
+Version: 0.2.6
 Summary: Library to access and aggregate several Mozilla data sources.
 Home-page: https://github.com/mozilla/libmozdata
 Author: Mozilla Release Management

{libmozdata-0.2.3 → libmozdata-0.2.6}/libmozdata.egg-info/SOURCES.txt

@@ -11,6 +11,7 @@ libmozdata/HGFileInfo.py
 libmozdata/__init__.py
 libmozdata/bugzilla.py
 libmozdata/buildhub.py
+libmozdata/clouseau.py
 libmozdata/config.py
 libmozdata/connection.py
 libmozdata/fx_trains.py

libmozdata-0.2.3/VERSION

@@ -1 +0,0 @@
-0.2.3

{libmozdata-0.2.3 → libmozdata-0.2.6}/libmozdata/buildhub.py

@@ -6,9 +6,9 @@ import json
 import os
 import re
 from collections import defaultdict
+from distutils.version import LooseVersion
 
 import requests
-from distutils.version import LooseVersion
 
 SEARCH_URL = os.getenv("BUILDHUB_SEARCH_URL", "https://buildhub.moz.tools/api/search")