squad 1.91__tar.gz → 1.93__tar.gz

{squad-1.91 → squad-1.93}/CHANGELOG.md

@@ -1,3 +1,11 @@
+# 1.93
+
+This 1.93 release adds support for a build log parser as well as some other minor bug fixes and improvements.
+
+# 1.92
+
+This 1.92 release avoids an extra retry when fetching errored builds/tests in Tuxsuite.
+
 # 1.91
 
 This 1.91 release brings a few changes worth pointing out separately:

{squad-1.91/squad.egg-info → squad-1.93}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: squad
-Version: 1.91
+Version: 1.93
 Summary: Software Quality Dashboard
 Home-page: https://github.com/Linaro/squad
 Author: Antonio Terceiro

{squad-1.91 → squad-1.93}/squad/api/prometheus.py

@@ -43,7 +43,7 @@ def metrics(request):
 
         for queue in queues:
             if queue['name'] in available_queues:
-                metric_name = f'queue_{queue["name"]}_length'
+                metric_name = f'queue_{queue["name"]}_count'
                 length = queue['messages_ready']
 
                 output += f'\n# TYPE {metric_name} counter'

{squad-1.91 → squad-1.93}/squad/ci/backend/tuxsuite.py

@@ -10,6 +10,7 @@ from requests.adapters import HTTPAdapter, Retry
 from functools import reduce
 from urllib.parse import urljoin
 
+from cryptography.exceptions import InvalidSignature
 from cryptography.hazmat.primitives.asymmetric import ec
 from cryptography.hazmat.primitives import (
     hashes,
@@ -249,9 +250,6 @@ class Backend(BaseBackend):
         test_job.name = test_name
 
         build_status = results['build_status']
-        if build_status == 'error' and results['retry'] < 2:
-            # SQUAD should retry fetching the build until retry == 2
-            raise TemporaryFetchIssue(results.get('status_message', 'TuxSuite Error'))
 
         # Make metadata
         metadata_keys = settings.get('BUILD_METADATA_KEYS', [])
@@ -270,7 +268,7 @@ class Backend(BaseBackend):
         metrics = {}
 
         completed = True
-        if results['retry'] >= 2:
+        if build_status == 'error':
             # This indicates that TuxSuite gave up trying to work on this build
             status = 'Incomplete'
             tests[f'build/{test_name}'] = 'skip'
@@ -484,22 +482,18 @@ class Backend(BaseBackend):
         if public_key is None:
             raise Exception("missing tuxsuite public key for this project")
 
-        payload = json.loads(request.body)
         signature = base64.urlsafe_b64decode(signature)
         key = serialization.load_ssh_public_key(public_key.encode("ascii"))
-        key.verify(
-            signature,
-            payload.encode("utf-8"),
-            ec.ECDSA(hashes.SHA256()),
-        )
+        try:
+            key.verify(
+                signature,
+                request.body,
+                ec.ECDSA(hashes.SHA256()),
+            )
+        except InvalidSignature:
+            raise Exception("Failed to verify signature against payload")
 
     def process_callback(self, json_payload, build, environment, backend):
-        # The payload coming from Tuxsuite is formatted as bytes,
-        # so after the first json.loads(request.body), the result
-        # will still be a string containing the actual json document
-        # We need to call json.loads() once more to get the actual
-        # python dict containing all the information we need
-        json_payload = json.loads(json_payload)
         if "kind" not in json_payload or "status" not in json_payload:
             raise Exception("`kind` and `status` are required in the payload")
 

squad-1.93/squad/core/management/commands/run_log_parser.py

@@ -0,0 +1,44 @@
+from django.core.management.base import BaseCommand
+
+from squad.plugins.linux_log_parser import Plugin as BootTestLogParser
+from squad.plugins.linux_log_parser_build import Plugin as BuildLogParser
+
+
+class FakeTestRun:
+    log_file = None
+    id = None
+
+
+log_parsers = {
+    'linux_log_parser_boot_test': BootTestLogParser(),
+    "linux_log_parser_build": BuildLogParser(),
+}
+
+
+class Command(BaseCommand):
+
+    help = """Run a log parser and print the outputs to the stdout."""
+
+    def add_arguments(self, parser):
+
+        parser.add_argument(
+            "LOG_FILE",
+            help="Log file to parser",
+        )
+
+        parser.add_argument(
+            "LOG_PARSER",
+            choices=log_parsers.keys(),
+            help="Which log parser to run"
+        )
+
+    def handle(self, *args, **options):
+        self.options = options
+
+        with open(options["LOG_FILE"], "r") as f:
+            log_file = f.read()
+
+        testrun = FakeTestRun()
+        testrun.log_file = log_file
+        parser = log_parsers[options["LOG_PARSER"]]
+        parser.postprocess_testrun(testrun, squad=False, print=True)

{squad-1.91 → squad-1.93}/squad/core/models.py

@@ -577,12 +577,17 @@ class Build(models.Model):
         List of attachments from all testruns
         """
         if self.__attachments__ is None:
+            test_run_ids = self.test_runs.values_list('id', flat=True)
+            all_attachments = Attachment.objects.filter(test_run_id__in=test_run_ids).values(
+                'test_run_id', 'filename'
+            )
+
             attachments = {}
-            for test_run in self.test_runs.all():
-                attachments[test_run.pk] = []
-                for attachment in test_run.attachments.all():
-                    attachments[test_run.pk].append(attachment.filename)
+            for attachment in all_attachments:
+                attachments.setdefault(attachment['test_run_id'], []).append(attachment['filename'])
+
             self.__attachments__ = attachments
+
         return self.__attachments__
 
     @property

{squad-1.91 → squad-1.93}/squad/plugins/lib/base_log_parser.py

@@ -16,8 +16,16 @@ square_brackets_and_contents = r"\[[^\]]+\]"
 
 class BaseLogParser:
     def compile_regexes(self, regexes):
-        combined = [r"(%s)" % r[REGEX_BODY] for r in regexes]
-        return re.compile(r"|".join(combined), re.S | re.M)
+        with_brackets = [r"(%s)" % r[REGEX_BODY] for r in regexes]
+        combined = r"|".join(with_brackets)
+
+        # In the case where there is only one regex, we need to add extra
+        # bracket around it for it to behave the same as the multiple regex
+        # case
+        if len(regexes) == 1:
+            combined = f"({combined})"
+
+        return re.compile(combined, re.S | re.M)
 
     def remove_numbers_and_time(self, snippet):
         # [ 1067.461794][  T132] BUG: KCSAN: data-race in do_page_fault spectre_v4_enable_task_mitigation
@@ -26,7 +34,11 @@ class BaseLogParser:
 
         # [ .][  T] BUG: KCSAN: data-race in do_page_fault spectre_v_enable_task_mitigation
         # ->  BUG: KCSAN: data-race in do_page_fault spectre_v_enable_task_mitigation
-        without_time = re.sub(f"^{square_brackets_and_contents}({square_brackets_and_contents})?", "", without_numbers) # noqa
+        without_time = re.sub(
+            f"^{square_brackets_and_contents}({square_brackets_and_contents})?",
+            "",
+            without_numbers,
+        )  # noqa
 
         return without_time
 
@@ -41,10 +53,7 @@ class BaseLogParser:
         snippet = matches[0]
         without_numbers_and_time = self.remove_numbers_and_time(snippet)
 
-        # Limit the name length to 191 characters, since the max name length
-        # for SuiteMetadata in SQUAD is 256 characters. The SHA and "-" take 65
-        # characters: 256-65=191
-        return slugify(without_numbers_and_time)[:191]
+        return slugify(without_numbers_and_time)
 
     def create_shasum(self, snippet):
         sha = hashlib.sha256()
@@ -52,7 +61,7 @@ class BaseLogParser:
         sha.update(without_numbers_and_time.encode())
         return sha.hexdigest()
 
-    def create_name_log_dict(self, test_name, lines, test_regex=None):
+    def create_name_log_dict(self, test_name, lines, test_regex=None, create_shas=True):
         """
         Produce a dictionary with the test names as keys and the extracted logs
         for that test name as values. There will be at least one test name per
@@ -64,31 +73,43 @@ class BaseLogParser:
         # have any output for a particular regex, just use the default name
         # (for example "check-kernel-oops").
         tests_without_shas_to_create = defaultdict(set)
-        tests_with_shas_to_create = defaultdict(set)
+        tests_with_shas_to_create = None
 
         # If there are lines, then create the tests for these.
         for line in lines:
             extracted_name = self.create_name(line, test_regex)
             if extracted_name:
-                extended_test_name = f"{test_name}-{extracted_name}"
+                max_name_length = 256
+                # If adding SHAs, limit the name length to 191 characters,
+                # since the max name length for SuiteMetadata in SQUAD is 256
+                # characters. The SHA and "-" take 65 characters: 256-65=191
+                if create_shas:
+                    max_name_length -= 65
+                extended_test_name = f"{test_name}-{extracted_name}"[:max_name_length]
             else:
                 extended_test_name = test_name
             tests_without_shas_to_create[extended_test_name].add(line)
 
-        for name, test_lines in tests_without_shas_to_create.items():
-            # Some lines of the matched regex might be the same, and we don't want to create
-            # multiple tests like test1-sha1, test1-sha1, etc, so we'll create a set of sha1sums
-            # then create only new tests for unique sha's
+        if create_shas:
+            tests_with_shas_to_create = defaultdict(set)
+            for name, test_lines in tests_without_shas_to_create.items():
+                # Some lines of the matched regex might be the same, and we don't want to create
+                # multiple tests like test1-sha1, test1-sha1, etc, so we'll create a set of sha1sums
+                # then create only new tests for unique sha's
 
-            for line in test_lines:
-                sha = self.create_shasum(line)
-                name_with_sha = f"{name}-{sha}"
-                tests_with_shas_to_create[name_with_sha].add(line)
+                for line in test_lines:
+                    sha = self.create_shasum(line)
+                    name_with_sha = f"{name}-{sha}"
+                    tests_with_shas_to_create[name_with_sha].add(line)
 
         return tests_without_shas_to_create, tests_with_shas_to_create
 
     def create_squad_tests_from_name_log_dict(
-        self, suite, testrun, tests_without_shas_to_create, tests_with_shas_to_create
+        self,
+        suite_name,
+        testrun,
+        tests_without_shas_to_create,
+        tests_with_shas_to_create=None,
     ):
         # Import SuiteMetadata from SQUAD only when required so BaseLogParser
         # does not require a SQUAD to work. This makes it easier to reuse this
@@ -96,6 +117,8 @@ class BaseLogParser:
         # patterns.
         from squad.core.models import SuiteMetadata
 
+        suite, _ = testrun.build.project.suites.get_or_create(slug=suite_name)
+
         for name, lines in tests_without_shas_to_create.items():
             metadata, _ = SuiteMetadata.objects.get_or_create(
                 suite=suite.slug, name=name, kind="test"
@@ -108,34 +131,72 @@ class BaseLogParser:
                 build=testrun.build,
                 environment=testrun.environment,
             )
-        for name_with_sha, lines in tests_with_shas_to_create.items():
-            metadata, _ = SuiteMetadata.objects.get_or_create(
-                suite=suite.slug, name=name_with_sha, kind="test"
-            )
-            testrun.tests.create(
-                suite=suite,
-                result=False,
-                log="\n---\n".join(lines),
-                metadata=metadata,
-                build=testrun.build,
-                environment=testrun.environment,
-            )
-
-    def create_squad_tests(self, testrun, suite, test_name, lines, test_regex=None):
+        if tests_with_shas_to_create:
+            for name_with_sha, lines in tests_with_shas_to_create.items():
+                metadata, _ = SuiteMetadata.objects.get_or_create(
+                    suite=suite.slug, name=name_with_sha, kind="test"
+                )
+                testrun.tests.create(
+                    suite=suite,
+                    result=False,
+                    log="\n---\n".join(lines),
+                    metadata=metadata,
+                    build=testrun.build,
+                    environment=testrun.environment,
+                )
+
+    def print_squad_tests_from_name_log_dict(
+        self,
+        suite_name,
+        tests_without_shas_to_create,
+        tests_with_shas_to_create=None,
+    ):
+        for name, lines in tests_without_shas_to_create.items():
+            print(f"\nName: {suite_name}/{name}")
+            log = "\n".join(lines)
+            print(f"Log:\n{log}")
+
+        if tests_with_shas_to_create:
+            for name_with_sha, lines in tests_with_shas_to_create.items():
+                print(f"\nName: {suite_name}/{name_with_sha}")
+                log = "\n---\n".join(lines)
+                print(f"Log:\n{log}")
+
+    def create_squad_tests(
+        self,
+        testrun,
+        suite_name,
+        test_name,
+        lines,
+        test_regex=None,
+        create_shas=True,
+        print=False,
+        squad=True,
+    ):
         """
         There will be at least one test per regex. If there were any match for
         a given regex, then a new test will be generated using test_name +
         shasum. This helps comparing kernel logs across different builds
         """
+
         tests_without_shas_to_create, tests_with_shas_to_create = (
-            self.create_name_log_dict(test_name, lines, test_regex)
-        )
-        self.create_squad_tests_from_name_log_dict(
-            suite,
-            testrun,
-            tests_without_shas_to_create,
-            tests_with_shas_to_create,
+            self.create_name_log_dict(
+                test_name, lines, test_regex, create_shas=create_shas
+            )
         )
+        if print:
+            self.print_squad_tests_from_name_log_dict(
+                suite_name,
+                tests_without_shas_to_create,
+                tests_with_shas_to_create,
+            )
+        if squad:
+            self.create_squad_tests_from_name_log_dict(
+                suite_name,
+                testrun,
+                tests_without_shas_to_create,
+                tests_with_shas_to_create,
+            )
 
     def join_matches(self, matches, regexes):
         """

squad-1.93/squad/plugins/linux_log_parser.py

@@ -0,0 +1,72 @@
+import logging
+import re
+from squad.plugins import Plugin as BasePlugin
+from squad.plugins.lib.base_log_parser import BaseLogParser, REGEX_NAME, REGEX_EXTRACT_NAME, tstamp, pid, not_newline_or_plus
+
+logger = logging.getLogger()
+
+MULTILINERS = [
+    ('exception', fr'-+\[? cut here \]?-+.*?{tstamp}{pid}?\s+-+\[? end trace \w* \]?-+', fr"\n{tstamp}{not_newline_or_plus}*"), # noqa
+    ('kasan', fr'{tstamp}{pid}?\s+=+\n{tstamp}{pid}?\s+BUG: KASAN:.*?\n*?{tstamp}{pid}?\s+=+', fr"BUG: KASAN:{not_newline_or_plus}*"), # noqa
+    ('kcsan', fr'{tstamp}{pid}?\s+=+\n{tstamp}{pid}?\s+BUG: KCSAN:.*?=+', fr"BUG: KCSAN:{not_newline_or_plus}*"), # noqa
+    ('kfence', fr'{tstamp}{pid}?\s+=+\n{tstamp}{pid}?\s+BUG: KFENCE:.*?{tstamp}{pid}?\s+=+', fr"BUG: KFENCE:{not_newline_or_plus}*"), # noqa
+    ('panic-multiline', fr'{tstamp}{pid}?\s+Kernel panic - [^\n]+\n.*?-+\[? end Kernel panic - [^\n]+ \]?-*', fr"Kernel {not_newline_or_plus}*"), # noqa
+    ('internal-error-oops', fr'{tstamp}{pid}?\s+Internal error: Oops.*?-+\[? end trace \w+ \]?-+', fr"Oops{not_newline_or_plus}*"), # noqa
+]
+
+ONELINERS = [
+    ('oops', r'^[^\n]+Oops(?: -|:).*?$', fr"Oops{not_newline_or_plus}*"), # noqa
+    ('fault', r'^[^\n]+Unhandled fault.*?$', fr"Unhandled {not_newline_or_plus}*"), # noqa
+    ('warning', r'^[^\n]+WARNING:.*?$', fr"WARNING:{not_newline_or_plus}*"), # noqa
+    ('bug', r'^[^\n]+(?: kernel BUG at|BUG:).*?$', fr"BUG{not_newline_or_plus}*"), # noqa
+    ('invalid-opcode', r'^[^\n]+invalid opcode:.*?$', fr"invalid opcode:{not_newline_or_plus}*"), # noqa
+    ('panic', r'Kernel panic - not syncing.*?$', fr"Kernel {not_newline_or_plus}*"), # noqa
+]
+
+# Tip: broader regexes should come first
+REGEXES = MULTILINERS + ONELINERS
+
+
+class Plugin(BasePlugin, BaseLogParser):
+    def __cutoff_boot_log(self, log):
+        # Attempt to split the log in " login:"
+        logs = log.split(' login:', 1)
+
+        # 1 string means no split was done, consider all logs as test log
+        if len(logs) == 1:
+            return '', log
+
+        boot_log = logs[0]
+        test_log = logs[1]
+        return boot_log, test_log
+
+    def __kernel_msgs_only(self, log):
+        kernel_msgs = re.findall(f'({tstamp}{pid}? .*?)$', log, re.S | re.M) # noqa
+        return '\n'.join(kernel_msgs)
+
+    def postprocess_testrun(self, testrun, squad=True, print=False):
+        # If running as a SQUAD plugin, only run the boot/test log parser if this is not a build testrun
+        if testrun.log_file is None or (squad and testrun.tests.filter(suite__slug="build").exists()):
+            return
+
+        boot_log, test_log = self.__cutoff_boot_log(testrun.log_file)
+        logs = {
+            'boot': boot_log,
+            'test': test_log,
+        }
+
+        for log_type, log in logs.items():
+            log = self.__kernel_msgs_only(log)
+            suite_name = f'log-parser-{log_type}'
+
+            regex = self.compile_regexes(REGEXES)
+            matches = regex.findall(log)
+            snippets = self.join_matches(matches, REGEXES)
+
+            for regex_id in range(len(REGEXES)):
+                test_name = REGEXES[regex_id][REGEX_NAME]
+                regex_pattern = REGEXES[regex_id][REGEX_EXTRACT_NAME]
+                test_name_regex = None
+                if regex_pattern:
+                    test_name_regex = re.compile(regex_pattern, re.S | re.M)
+                self.create_squad_tests(testrun, suite_name, test_name, snippets[regex_id], test_name_regex, squad=squad, print=print)