squad 1.91__py3-none-any.whl → 1.93__py3-none-any.whl

squad/api/prometheus.py

@@ -43,7 +43,7 @@ def metrics(request):
 
         for queue in queues:
             if queue['name'] in available_queues:
-                metric_name = f'queue_{queue["name"]}_length'
+                metric_name = f'queue_{queue["name"]}_count'
                 length = queue['messages_ready']
 
                 output += f'\n# TYPE {metric_name} counter'

squad/ci/backend/tuxsuite.py

@@ -10,6 +10,7 @@ from requests.adapters import HTTPAdapter, Retry
 from functools import reduce
 from urllib.parse import urljoin
 
+from cryptography.exceptions import InvalidSignature
 from cryptography.hazmat.primitives.asymmetric import ec
 from cryptography.hazmat.primitives import (
     hashes,
@@ -249,9 +250,6 @@ class Backend(BaseBackend):
         test_job.name = test_name
 
         build_status = results['build_status']
-        if build_status == 'error' and results['retry'] < 2:
-            # SQUAD should retry fetching the build until retry == 2
-            raise TemporaryFetchIssue(results.get('status_message', 'TuxSuite Error'))
 
         # Make metadata
         metadata_keys = settings.get('BUILD_METADATA_KEYS', [])
@@ -270,7 +268,7 @@ class Backend(BaseBackend):
         metrics = {}
 
         completed = True
-        if results['retry'] >= 2:
+        if build_status == 'error':
             # This indicates that TuxSuite gave up trying to work on this build
             status = 'Incomplete'
             tests[f'build/{test_name}'] = 'skip'
@@ -484,22 +482,18 @@ class Backend(BaseBackend):
         if public_key is None:
             raise Exception("missing tuxsuite public key for this project")
 
-        payload = json.loads(request.body)
         signature = base64.urlsafe_b64decode(signature)
         key = serialization.load_ssh_public_key(public_key.encode("ascii"))
-        key.verify(
-            signature,
-            payload.encode("utf-8"),
-            ec.ECDSA(hashes.SHA256()),
-        )
+        try:
+            key.verify(
+                signature,
+                request.body,
+                ec.ECDSA(hashes.SHA256()),
+            )
+        except InvalidSignature:
+            raise Exception("Failed to verify signature against payload")
 
     def process_callback(self, json_payload, build, environment, backend):
-        # The payload coming from Tuxsuite is formatted as bytes,
-        # so after the first json.loads(request.body), the result
-        # will still be a string containing the actual json document
-        # We need to call json.loads() once more to get the actual
-        # python dict containing all the information we need
-        json_payload = json.loads(json_payload)
         if "kind" not in json_payload or "status" not in json_payload:
             raise Exception("`kind` and `status` are required in the payload")
 

squad/core/management/commands/run_log_parser.py

@@ -0,0 +1,44 @@
+from django.core.management.base import BaseCommand
+
+from squad.plugins.linux_log_parser import Plugin as BootTestLogParser
+from squad.plugins.linux_log_parser_build import Plugin as BuildLogParser
+
+
+class FakeTestRun:
+    log_file = None
+    id = None
+
+
+log_parsers = {
+    'linux_log_parser_boot_test': BootTestLogParser(),
+    "linux_log_parser_build": BuildLogParser(),
+}
+
+
+class Command(BaseCommand):
+
+    help = """Run a log parser and print the outputs to the stdout."""
+
+    def add_arguments(self, parser):
+
+        parser.add_argument(
+            "LOG_FILE",
+            help="Log file to parser",
+        )
+
+        parser.add_argument(
+            "LOG_PARSER",
+            choices=log_parsers.keys(),
+            help="Which log parser to run"
+        )
+
+    def handle(self, *args, **options):
+        self.options = options
+
+        with open(options["LOG_FILE"], "r") as f:
+            log_file = f.read()
+
+        testrun = FakeTestRun()
+        testrun.log_file = log_file
+        parser = log_parsers[options["LOG_PARSER"]]
+        parser.postprocess_testrun(testrun, squad=False, print=True)

squad/core/models.py

@@ -577,12 +577,17 @@ class Build(models.Model):
         List of attachments from all testruns
         """
         if self.__attachments__ is None:
+            test_run_ids = self.test_runs.values_list('id', flat=True)
+            all_attachments = Attachment.objects.filter(test_run_id__in=test_run_ids).values(
+                'test_run_id', 'filename'
+            )
+
             attachments = {}
-            for test_run in self.test_runs.all():
-                attachments[test_run.pk] = []
-                for attachment in test_run.attachments.all():
-                    attachments[test_run.pk].append(attachment.filename)
+            for attachment in all_attachments:
+                attachments.setdefault(attachment['test_run_id'], []).append(attachment['filename'])
+
             self.__attachments__ = attachments
+
         return self.__attachments__
 
     @property

squad/plugins/lib/base_log_parser.py

@@ -16,8 +16,16 @@ square_brackets_and_contents = r"\[[^\]]+\]"
 
 class BaseLogParser:
     def compile_regexes(self, regexes):
-        combined = [r"(%s)" % r[REGEX_BODY] for r in regexes]
-        return re.compile(r"|".join(combined), re.S | re.M)
+        with_brackets = [r"(%s)" % r[REGEX_BODY] for r in regexes]
+        combined = r"|".join(with_brackets)
+
+        # In the case where there is only one regex, we need to add extra
+        # bracket around it for it to behave the same as the multiple regex
+        # case
+        if len(regexes) == 1:
+            combined = f"({combined})"
+
+        return re.compile(combined, re.S | re.M)
 
     def remove_numbers_and_time(self, snippet):
         # [ 1067.461794][  T132] BUG: KCSAN: data-race in do_page_fault spectre_v4_enable_task_mitigation
@@ -26,7 +34,11 @@ class BaseLogParser:
 
         # [ .][  T] BUG: KCSAN: data-race in do_page_fault spectre_v_enable_task_mitigation
         # ->  BUG: KCSAN: data-race in do_page_fault spectre_v_enable_task_mitigation
-        without_time = re.sub(f"^{square_brackets_and_contents}({square_brackets_and_contents})?", "", without_numbers) # noqa
+        without_time = re.sub(
+            f"^{square_brackets_and_contents}({square_brackets_and_contents})?",
+            "",
+            without_numbers,
+        )  # noqa
 
         return without_time
 
@@ -41,10 +53,7 @@ class BaseLogParser:
         snippet = matches[0]
         without_numbers_and_time = self.remove_numbers_and_time(snippet)
 
-        # Limit the name length to 191 characters, since the max name length
-        # for SuiteMetadata in SQUAD is 256 characters. The SHA and "-" take 65
-        # characters: 256-65=191
-        return slugify(without_numbers_and_time)[:191]
+        return slugify(without_numbers_and_time)
 
     def create_shasum(self, snippet):
         sha = hashlib.sha256()
@@ -52,7 +61,7 @@ class BaseLogParser:
         sha.update(without_numbers_and_time.encode())
         return sha.hexdigest()
 
-    def create_name_log_dict(self, test_name, lines, test_regex=None):
+    def create_name_log_dict(self, test_name, lines, test_regex=None, create_shas=True):
         """
         Produce a dictionary with the test names as keys and the extracted logs
         for that test name as values. There will be at least one test name per
@@ -64,31 +73,43 @@ class BaseLogParser:
         # have any output for a particular regex, just use the default name
         # (for example "check-kernel-oops").
         tests_without_shas_to_create = defaultdict(set)
-        tests_with_shas_to_create = defaultdict(set)
+        tests_with_shas_to_create = None
 
         # If there are lines, then create the tests for these.
         for line in lines:
             extracted_name = self.create_name(line, test_regex)
             if extracted_name:
-                extended_test_name = f"{test_name}-{extracted_name}"
+                max_name_length = 256
+                # If adding SHAs, limit the name length to 191 characters,
+                # since the max name length for SuiteMetadata in SQUAD is 256
+                # characters. The SHA and "-" take 65 characters: 256-65=191
+                if create_shas:
+                    max_name_length -= 65
+                extended_test_name = f"{test_name}-{extracted_name}"[:max_name_length]
             else:
                 extended_test_name = test_name
             tests_without_shas_to_create[extended_test_name].add(line)
 
-        for name, test_lines in tests_without_shas_to_create.items():
-            # Some lines of the matched regex might be the same, and we don't want to create
-            # multiple tests like test1-sha1, test1-sha1, etc, so we'll create a set of sha1sums
-            # then create only new tests for unique sha's
+        if create_shas:
+            tests_with_shas_to_create = defaultdict(set)
+            for name, test_lines in tests_without_shas_to_create.items():
+                # Some lines of the matched regex might be the same, and we don't want to create
+                # multiple tests like test1-sha1, test1-sha1, etc, so we'll create a set of sha1sums
+                # then create only new tests for unique sha's
 
-            for line in test_lines:
-                sha = self.create_shasum(line)
-                name_with_sha = f"{name}-{sha}"
-                tests_with_shas_to_create[name_with_sha].add(line)
+                for line in test_lines:
+                    sha = self.create_shasum(line)
+                    name_with_sha = f"{name}-{sha}"
+                    tests_with_shas_to_create[name_with_sha].add(line)
 
         return tests_without_shas_to_create, tests_with_shas_to_create
 
     def create_squad_tests_from_name_log_dict(
-        self, suite, testrun, tests_without_shas_to_create, tests_with_shas_to_create
+        self,
+        suite_name,
+        testrun,
+        tests_without_shas_to_create,
+        tests_with_shas_to_create=None,
     ):
         # Import SuiteMetadata from SQUAD only when required so BaseLogParser
         # does not require a SQUAD to work. This makes it easier to reuse this
@@ -96,6 +117,8 @@ class BaseLogParser:
         # patterns.
         from squad.core.models import SuiteMetadata
 
+        suite, _ = testrun.build.project.suites.get_or_create(slug=suite_name)
+
         for name, lines in tests_without_shas_to_create.items():
             metadata, _ = SuiteMetadata.objects.get_or_create(
                 suite=suite.slug, name=name, kind="test"
@@ -108,34 +131,72 @@ class BaseLogParser:
                 build=testrun.build,
                 environment=testrun.environment,
             )
-        for name_with_sha, lines in tests_with_shas_to_create.items():
-            metadata, _ = SuiteMetadata.objects.get_or_create(
-                suite=suite.slug, name=name_with_sha, kind="test"
-            )
-            testrun.tests.create(
-                suite=suite,
-                result=False,
-                log="\n---\n".join(lines),
-                metadata=metadata,
-                build=testrun.build,
-                environment=testrun.environment,
-            )
-
-    def create_squad_tests(self, testrun, suite, test_name, lines, test_regex=None):
+        if tests_with_shas_to_create:
+            for name_with_sha, lines in tests_with_shas_to_create.items():
+                metadata, _ = SuiteMetadata.objects.get_or_create(
+                    suite=suite.slug, name=name_with_sha, kind="test"
+                )
+                testrun.tests.create(
+                    suite=suite,
+                    result=False,
+                    log="\n---\n".join(lines),
+                    metadata=metadata,
+                    build=testrun.build,
+                    environment=testrun.environment,
+                )
+
+    def print_squad_tests_from_name_log_dict(
+        self,
+        suite_name,
+        tests_without_shas_to_create,
+        tests_with_shas_to_create=None,
+    ):
+        for name, lines in tests_without_shas_to_create.items():
+            print(f"\nName: {suite_name}/{name}")
+            log = "\n".join(lines)
+            print(f"Log:\n{log}")
+
+        if tests_with_shas_to_create:
+            for name_with_sha, lines in tests_with_shas_to_create.items():
+                print(f"\nName: {suite_name}/{name_with_sha}")
+                log = "\n---\n".join(lines)
+                print(f"Log:\n{log}")
+
+    def create_squad_tests(
+        self,
+        testrun,
+        suite_name,
+        test_name,
+        lines,
+        test_regex=None,
+        create_shas=True,
+        print=False,
+        squad=True,
+    ):
         """
         There will be at least one test per regex. If there were any match for
         a given regex, then a new test will be generated using test_name +
         shasum. This helps comparing kernel logs across different builds
         """
+
         tests_without_shas_to_create, tests_with_shas_to_create = (
-            self.create_name_log_dict(test_name, lines, test_regex)
-        )
-        self.create_squad_tests_from_name_log_dict(
-            suite,
-            testrun,
-            tests_without_shas_to_create,
-            tests_with_shas_to_create,
+            self.create_name_log_dict(
+                test_name, lines, test_regex, create_shas=create_shas
+            )
         )
+        if print:
+            self.print_squad_tests_from_name_log_dict(
+                suite_name,
+                tests_without_shas_to_create,
+                tests_with_shas_to_create,
+            )
+        if squad:
+            self.create_squad_tests_from_name_log_dict(
+                suite_name,
+                testrun,
+                tests_without_shas_to_create,
+                tests_with_shas_to_create,
+            )
 
     def join_matches(self, matches, regexes):
         """

squad/plugins/linux_log_parser.py

@@ -6,21 +6,21 @@ from squad.plugins.lib.base_log_parser import BaseLogParser, REGEX_NAME, REGEX_E
 logger = logging.getLogger()
 
 MULTILINERS = [
-    ('exception', f'-+\[? cut here \]?-+.*?{tstamp}{pid}?\s+-+\[? end trace \w* \]?-+', f"\n{tstamp}{not_newline_or_plus}*"), # noqa
-    ('kasan', f'{tstamp}{pid}?\s+=+\n{tstamp}{pid}?\s+BUG: KASAN:.*?\n*?{tstamp}{pid}?\s+=+', f"BUG: KASAN:{not_newline_or_plus}*"), # noqa
-    ('kcsan', f'{tstamp}{pid}?\s+=+\n{tstamp}{pid}?\s+BUG: KCSAN:.*?=+', f"BUG: KCSAN:{not_newline_or_plus}*"), # noqa
-    ('kfence', f'{tstamp}{pid}?\s+=+\n{tstamp}{pid}?\s+BUG: KFENCE:.*?{tstamp}{pid}?\s+=+', f"BUG: KFENCE:{not_newline_or_plus}*"), # noqa
-    ('panic-multiline', f'{tstamp}{pid}?\s+Kernel panic - [^\n]+\n.*?-+\[? end Kernel panic - [^\n]+ \]?-*', f"Kernel {not_newline_or_plus}*"), # noqa
-    ('internal-error-oops', f'{tstamp}{pid}?\s+Internal error: Oops.*?-+\[? end trace \w+ \]?-+', f"Oops{not_newline_or_plus}*"), # noqa
+    ('exception', fr'-+\[? cut here \]?-+.*?{tstamp}{pid}?\s+-+\[? end trace \w* \]?-+', fr"\n{tstamp}{not_newline_or_plus}*"), # noqa
+    ('kasan', fr'{tstamp}{pid}?\s+=+\n{tstamp}{pid}?\s+BUG: KASAN:.*?\n*?{tstamp}{pid}?\s+=+', fr"BUG: KASAN:{not_newline_or_plus}*"), # noqa
+    ('kcsan', fr'{tstamp}{pid}?\s+=+\n{tstamp}{pid}?\s+BUG: KCSAN:.*?=+', fr"BUG: KCSAN:{not_newline_or_plus}*"), # noqa
+    ('kfence', fr'{tstamp}{pid}?\s+=+\n{tstamp}{pid}?\s+BUG: KFENCE:.*?{tstamp}{pid}?\s+=+', fr"BUG: KFENCE:{not_newline_or_plus}*"), # noqa
+    ('panic-multiline', fr'{tstamp}{pid}?\s+Kernel panic - [^\n]+\n.*?-+\[? end Kernel panic - [^\n]+ \]?-*', fr"Kernel {not_newline_or_plus}*"), # noqa
+    ('internal-error-oops', fr'{tstamp}{pid}?\s+Internal error: Oops.*?-+\[? end trace \w+ \]?-+', fr"Oops{not_newline_or_plus}*"), # noqa
 ]
 
 ONELINERS = [
-    ('oops', r'^[^\n]+Oops(?: -|:).*?$', f"Oops{not_newline_or_plus}*"), # noqa
-    ('fault', r'^[^\n]+Unhandled fault.*?$', f"Unhandled {not_newline_or_plus}*"), # noqa
-    ('warning', r'^[^\n]+WARNING:.*?$', f"WARNING:{not_newline_or_plus}*"), # noqa
-    ('bug', r'^[^\n]+(?: kernel BUG at|BUG:).*?$', f"BUG{not_newline_or_plus}*"), # noqa
-    ('invalid-opcode', r'^[^\n]+invalid opcode:.*?$', f"invalid opcode:{not_newline_or_plus}*"), # noqa
-    ('panic', r'Kernel panic - not syncing.*?$', f"Kernel {not_newline_or_plus}*"), # noqa
+    ('oops', r'^[^\n]+Oops(?: -|:).*?$', fr"Oops{not_newline_or_plus}*"), # noqa
+    ('fault', r'^[^\n]+Unhandled fault.*?$', fr"Unhandled {not_newline_or_plus}*"), # noqa
+    ('warning', r'^[^\n]+WARNING:.*?$', fr"WARNING:{not_newline_or_plus}*"), # noqa
+    ('bug', r'^[^\n]+(?: kernel BUG at|BUG:).*?$', fr"BUG{not_newline_or_plus}*"), # noqa
+    ('invalid-opcode', r'^[^\n]+invalid opcode:.*?$', fr"invalid opcode:{not_newline_or_plus}*"), # noqa
+    ('panic', r'Kernel panic - not syncing.*?$', fr"Kernel {not_newline_or_plus}*"), # noqa
 ]
 
 # Tip: broader regexes should come first
@@ -44,8 +44,9 @@ class Plugin(BasePlugin, BaseLogParser):
         kernel_msgs = re.findall(f'({tstamp}{pid}? .*?)$', log, re.S | re.M) # noqa
         return '\n'.join(kernel_msgs)
 
-    def postprocess_testrun(self, testrun):
-        if testrun.log_file is None:
+    def postprocess_testrun(self, testrun, squad=True, print=False):
+        # If running as a SQUAD plugin, only run the boot/test log parser if this is not a build testrun
+        if testrun.log_file is None or (squad and testrun.tests.filter(suite__slug="build").exists()):
             return
 
         boot_log, test_log = self.__cutoff_boot_log(testrun.log_file)
@@ -56,7 +57,7 @@ class Plugin(BasePlugin, BaseLogParser):
 
         for log_type, log in logs.items():
             log = self.__kernel_msgs_only(log)
-            suite, _ = testrun.build.project.suites.get_or_create(slug=f'log-parser-{log_type}')
+            suite_name = f'log-parser-{log_type}'
 
             regex = self.compile_regexes(REGEXES)
             matches = regex.findall(log)
@@ -68,4 +69,4 @@ class Plugin(BasePlugin, BaseLogParser):
                 test_name_regex = None
                 if regex_pattern:
                     test_name_regex = re.compile(regex_pattern, re.S | re.M)
-                self.create_squad_tests(testrun, suite, test_name, snippets[regex_id], test_name_regex)
+                self.create_squad_tests(testrun, suite_name, test_name, snippets[regex_id], test_name_regex, squad=squad, print=print)

squad/plugins/linux_log_parser_build.py

@@ -0,0 +1,332 @@
+import logging
+import re
+
+from django.template.defaultfilters import slugify
+
+from squad.plugins import Plugin as BasePlugin
+from squad.plugins.lib.base_log_parser import (
+    REGEX_EXTRACT_NAME,
+    REGEX_NAME,
+    BaseLogParser,
+)
+
+logger = logging.getLogger()
+
+file_path = r"^(?:[^\n]*?:(?:\d+:){2}|<[^\n]*?>:)"
+gcc_clang_compiler_error_warning = rf"{file_path} (?:error|warning): [^\n]+?\n^(?:\.+\n|^(?!\s+(?:CC|Kernel[^\n]*?is ready))\s+?[^\n]+\n|{file_path} note:[^\n]+\n)*"
+
+MULTILINERS_GCC = [
+    (
+        "gcc-compiler",
+        gcc_clang_compiler_error_warning,
+        r"^[^\n]*(?:error|warning)[^\n]*$",
+    ),
+]
+
+ONELINERS_GCC = []
+
+
+MULTILINERS_CLANG = [
+    (
+        "clang-compiler",
+        gcc_clang_compiler_error_warning,
+        r"^[^\n]*(?:error|warning)[^\n]*$",
+    ),
+]
+
+ONELINERS_CLANG = [
+    (
+        "clang-compiler-single-line",
+        "^clang: (?:error|warning).*?$",
+        r"^[^\n]*(?:error|warning).*?$",
+    ),
+    (
+        "clang-compiler-fatal-error",
+        "^fatal error.*?$",
+        r"^fatal error.*?$",
+    ),
+]
+
+MULTILINERS_GENERAL = [
+    (
+        "general-not-a-git-repo",
+        r"^[^\n]*fatal: not a git repository.*?not set\)\.$",
+        r"^[^\n]*fatal: not a git repository.*?$",
+    ),
+    (
+        "general-unexpected-argument",
+        r"^[^\n]*error: Found argument.*?--help$",
+        r"^[^\n]*error: Found argument.*?$",
+    ),
+    (
+        "general-broken-32-bit",
+        r"^[^\n]*Warning: you seem to have a broken 32-bit build.*?(?:If[^\n]*?try:(?:\n|\s+.+?$)+)+",
+        r"^[^\n]*Warning:.*?$",
+    ),
+    (
+        "general-makefile-overriding",
+        r"^[^\n]*warning: overriding recipe for target.*?ignoring old recipe for target.*?$",
+        r"^[^\n]*warning:.*?$",
+    ),
+    (
+        "general-unmet-dependencies",
+        r"^WARNING: unmet direct dependencies detected for.*?$(?:\n +[^\n]+)*",
+        r"^WARNING: unmet direct dependencies detected for.*?$",
+    ),
+    (
+        "general-ldd",
+        r"^[^\n]*?lld:[^\n]+?(?:warning|error):.*?$(?:\n^>>>[^\n]+)*",
+        r"^[^\n]*?lld:.*?$",
+    ),
+    (
+        "general-ld",
+        r"^[^\n]*?ld:[^\n]+?(?:warning|error):[^\n]*?$(?:\n^[^\n]*?NOTE:[^\n]+)*",
+        r"^[^\n]*?ld:[^\n]+?(?:warning|error):.*?$",
+    ),
+    (
+        "general-objcopy",
+        r"^[^\n]*?objcopy:[^\n]+?(?:warning|error):[^\n]*?$(?:\n^[^\n]*?NOTE:[^\n]+)*",
+        r"^[^\n]*?objcopy:[^\n]+?(?:warning|error):.*?$",
+    ),
+    (
+        "general-ld-undefined-reference",
+        r"^[^\n]*?ld[^\n]*?$\n^[^\n]+undefined reference.*?$",
+        r"^[^\n]+undefined reference.*?$",
+    ),
+    (
+        "general-modpost",
+        r"^[^\n]*?WARNING: modpost:[^\n]*?$(?:\n^To see.*?:$\n^.*?$)?",
+        r"^[^\n]*?WARNING.*?$",
+    ),
+    (
+        "general-python-traceback",
+        r"Traceback.*?^[^\s]+Error: .*?$",
+        r"^[^\s]+Error: .*?$",
+    ),
+]
+
+ONELINERS_GENERAL = [
+    (
+        "general-no-such-file-or-directory",
+        r"^[^\n]+?No such file or directory.*?$",
+        r"^[^\n]+?No such file or directory.*?$",
+    ),
+    (
+        "general-no-targets",
+        r"^[^\n]+?No targets.*?$",
+        r"^[^\n]+?No targets.*?$",
+    ),
+    (
+        "general-no-rule-to-make-target",
+        r"^[^\n]+?No rule to make target.*?$",
+        r"^[^\n]+?No rule to make target.*?$",
+    ),
+    (
+        "general-makefile-config",
+        r"^Makefile.config:\d+:.*?$",
+        r"^Makefile.config:\d+:.*?$",
+    ),
+    (
+        "general-not-found",
+        r"^[^\n]*?not found.*?$",
+        r"^[^\n]*?not found.*?$",
+    ),
+    (
+        "general-kernel-abi",
+        r"^Warning: Kernel ABI header at.*?$",
+        r"^Warning: Kernel ABI header at.*?$",
+    ),
+    (
+        "general-missing",
+        r"^Warning: missing.*?$",
+        r"^Warning: missing.*?$",
+    ),
+    (
+        "general-dtc",
+        r"^[^\n]*?Warning \([^\n]*?\).*?$",
+        r"^[^\n]*?Warning.*?$",
+    ),
+    (
+        "general-register-allocation",
+        r"^[^\n]*?error: register allocation failed.*?$",
+        r"^[^\n]*?error.*?$",
+    ),
+]
+
+# Tip: broader regexes should come first
+REGEXES_GCC = MULTILINERS_GCC + MULTILINERS_GENERAL + ONELINERS_GCC + ONELINERS_GENERAL
+REGEXES_CLANG = (
+    MULTILINERS_CLANG + MULTILINERS_GENERAL + ONELINERS_CLANG + ONELINERS_GENERAL
+)
+
+supported_toolchains = {
+    "gcc": REGEXES_GCC,
+    "clang": REGEXES_CLANG,
+}
+
+make_regex = r"^make .*?$"
+in_file_regex = r"^In file[^\n]*?[:,]$(?:\n^(?:\s+|In file)[^\n]*?[:,]$)*"
+in_function_regex = r"^[^\n]*?In function.*?:$"
+entering_dir_regex = r"^make\[(?:\d+)\]: Entering directory.*?$"
+leaving_dir_regex = r"^make\[(?:\d+)\]: Leaving directory.*?$"
+
+split_regex_gcc = rf"(.*?)({make_regex}|{in_file_regex}|{in_function_regex}|{entering_dir_regex}|{leaving_dir_regex})"
+
+
+class Plugin(BasePlugin, BaseLogParser):
+
+    def post_process_test_name(self, text):
+        # Remove "builds/linux" if there
+        text = re.sub(r"builds/linux", "", text)
+
+        # Change "/" and "." to "_" for readability
+        text = re.sub(r"[/\.]", "_", text)
+
+        # Remove numbers and hex
+        text = re.sub(r"(0x[a-f0-9]+|[<\[][0-9a-f]+?[>\]]|\d+)", "", text)
+
+        # Remove "{...}" and "[...]"
+        text = re.sub(r"\{.+?\}", "", text)
+        text = re.sub(r"\[.+?\]", "", text)
+
+        return text
+
+    def create_name(self, snippet, compiled_regex=None):
+        matches = None
+        if compiled_regex:
+            matches = compiled_regex.findall(snippet)
+        if not matches:
+            # Only extract a name if we provide a regex to extract the name and
+            # there is a match
+            return None
+        snippet = matches[0]
+        without_numbers_and_time = self.remove_numbers_and_time(snippet)
+
+        name = slugify(self.post_process_test_name(without_numbers_and_time))
+
+        return name
+
+    def split_by_regex(self, log, regex):
+        # Split up the log by the keywords we want to capture
+        s_lines_compiled = re.compile(regex, re.DOTALL | re.MULTILINE)
+        split_by_regex_list = s_lines_compiled.split(log)
+        split_by_regex_list = [
+            f for f in split_by_regex_list if f is not None and f != ""
+        ]
+
+        return split_by_regex_list
+
+    def process_blocks(
+        self,
+        blocks_to_process,
+        regexes,
+        make_regex=make_regex,
+        entering_dir_regex=entering_dir_regex,
+        leaving_dir_regex=leaving_dir_regex,
+        in_file_regex=in_file_regex,
+        in_function_regex=in_function_regex,
+    ):
+        snippets = dict()
+        regex_compiled = self.compile_regexes(regexes)
+        make_regex_compiled = re.compile(make_regex, re.DOTALL | re.MULTILINE)
+        entering_dir_regex_compiled = re.compile(
+            entering_dir_regex, re.DOTALL | re.MULTILINE
+        )
+        leaving_dir_regex_compiled = re.compile(
+            leaving_dir_regex, re.DOTALL | re.MULTILINE
+        )
+        in_file_regex_compiled = re.compile(in_file_regex, re.DOTALL | re.MULTILINE)
+        in_function_regex_compiled = re.compile(
+            in_function_regex, re.DOTALL | re.MULTILINE
+        )
+
+        # For tracking the last piece of information we saw
+        make_command = None
+        entering_dir = None
+        in_file = None
+        in_function = None
+
+        for regex_id in range(len(regexes)):
+            snippets[regex_id] = []
+        for block in blocks_to_process:
+            if make_regex_compiled.match(block):
+                make_command = block
+                entering_dir = None
+                in_file = None
+                in_function = None
+            elif entering_dir_regex_compiled.match(block):
+                entering_dir = block
+                in_file = None
+                in_function = None
+            elif leaving_dir_regex_compiled.match(block):
+                entering_dir = None
+                in_file = None
+                in_function = None
+            elif in_file_regex_compiled.match(block):
+                in_file = block
+                in_function = None
+            elif in_function_regex_compiled.match(block):
+                in_function = block
+            else:
+                matches = regex_compiled.findall(block)
+                sub_snippets = self.join_matches(matches, regexes)
+                prepend = ""
+                if make_command:
+                    prepend += make_command + "\n"
+                if entering_dir:
+                    prepend += entering_dir + "\n"
+                if in_file:
+                    prepend += in_file + "\n"
+                if in_function:
+                    prepend += in_function + "\n"
+                for regex_id in range(len(regexes)):
+                    for s in sub_snippets[regex_id]:
+                        snippets[regex_id].append(prepend + s)
+
+        return snippets
+
+    def postprocess_testrun(self, testrun, squad=True, print=False):
+        """
+        Check:
+            - There is a log file
+            - If running as SQUAD plugin, the testrun contains the "build"
+              suite - this tells us that the testrun's log is a build log
+        """
+        if testrun.log_file is None or (
+            squad and not testrun.tests.filter(suite__slug="build").exists()
+        ):
+            return
+
+        regexes = None
+        for toolchain, toolchain_regexes in supported_toolchains.items():
+            if f"--toolchain={toolchain}" in testrun.log_file:
+                toolchain_name = toolchain
+                regexes = toolchain_regexes
+
+        # If a supported toolchain was not found in the log
+        if regexes is None:
+            return
+
+        # If running in SQUAD, create the suite
+        suite_name = f"log-parser-build-{toolchain_name}"
+
+        blocks_to_process = self.split_by_regex(testrun.log_file, split_regex_gcc)
+
+        snippets = self.process_blocks(blocks_to_process, regexes)
+
+        for regex_id in range(len(regexes)):
+            test_name = regexes[regex_id][REGEX_NAME]
+            regex_pattern = regexes[regex_id][REGEX_EXTRACT_NAME]
+            test_name_regex = None
+            if regex_pattern:
+                test_name_regex = re.compile(regex_pattern, re.S | re.M)
+            self.create_squad_tests(
+                testrun,
+                suite_name,
+                test_name,
+                snippets[regex_id],
+                test_name_regex,
+                create_shas=False,
+                print=print,
+                squad=squad,
+            )

squad/version.py

@@ -1 +1 @@
-__version__ = '1.91'
+__version__ = '1.93'

{squad-1.91.dist-info → squad-1.93.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: squad
-Version: 1.91
+Version: 1.93
 Summary: Software Quality Dashboard
 Home-page: https://github.com/Linaro/squad
 Author: Antonio Terceiro

{squad-1.91.dist-info → squad-1.93.dist-info}/RECORD

@@ -10,14 +10,14 @@ squad/manage.py,sha256=Z-LXT67p0R-IzwJ9fLIAacEZmU0VUjqDOSg7j2ZSxJ4,1437
 squad/settings.py,sha256=0MZ48SV_7CTrLMik2ubWf8-ROQiFju6CKnUC3iR8KAc,14800
 squad/socialaccount.py,sha256=vySqPwQ3qVVpahuJ-Snln8K--yzRL3bw4Nx27AsB39A,789
 squad/urls.py,sha256=JiEfVW8YlzLPE52c2aHzdn5kVVKK4o22w8h5KOA6QhQ,2776
-squad/version.py,sha256=S8qXLXebPToWZIRCvA5VIWk_5c_MVZR49-YdwQ0ypzc,21
+squad/version.py,sha256=K6dPzAHfR0_z4DOLvpBamlquLtpey9vWxxcBlvLvrn4,21
 squad/wsgi.py,sha256=SF8T0cQ0OPVyuYjO5YXBIQzvSXQHV0M2BTmd4gP1rPs,387
 squad/api/__init__.py,sha256=CJiVakfAlHVN5mIFRVQYZQfuNUhUgWVbsdYTME4tq7U,1349
 squad/api/apps.py,sha256=Trk72p-iV1uGn0o5mdJn5HARUoHGbfgO49jwXvpkmdQ,141
 squad/api/ci.py,sha256=QjGIhSpm8gmIjH4Nd2NAWtJItSVleg3QOLxBU_p9h1E,7082
 squad/api/data.py,sha256=obKDV0-neEvj5lPF9VED2gy_hpfhGtLJABYvSY38ing,2379
 squad/api/filters.py,sha256=Zvp8DCJmiNquFWqvfVseEAAMYYPiT95RUjqKdzcqSnw,6917
-squad/api/prometheus.py,sha256=0usJgOz14g1a71sdfjM-cOC8IGXkpE-5-TqpvJj-Oyk,1840
+squad/api/prometheus.py,sha256=MEzSZtYYx6PXreIwZzcVHLp-1vVQ9IKhi9hb8b0vjUk,1839
 squad/api/rest.py,sha256=ZtbK0c1BLPPnsX79XlKFVYONM_VJ0vacWZ2JsdCd4l0,77342
 squad/api/urls.py,sha256=c-o27_RP0ynOtxuyRKUl274fFMWWrzoii31Mr2saxSQ,1414
 squad/api/utils.py,sha256=Sa8QFId3_oSqD2UOoY3Kuh54LLDLPNMq2sub5ktd6Fs,1160
@@ -33,7 +33,7 @@ squad/ci/backend/__init__.py,sha256=yhpotXT9F4IdAOXvGQ3-17eOHAFwoaqf9SnMX17ab30,
 squad/ci/backend/fake.py,sha256=7Rl-JXnBYThDomOBzBsN9XuVkSjSHTZjtZOURdowZbA,2397
 squad/ci/backend/lava.py,sha256=WeOJJNxv42geGf3Y6r-I0WnhWinxpSSgZAFAwfkiXGY,34039
 squad/ci/backend/null.py,sha256=htEd4NbrXLKdPgFfTS0Ixm8PdT6Ghat3BCYi2zjfuv0,5624
-squad/ci/backend/tuxsuite.py,sha256=HTYLyJvtraHnkMKOjYix66bq1QV4m8bamNBahV5SZZw,19129
+squad/ci/backend/tuxsuite.py,sha256=pFcNdcHpFzalHPQhbSY6ryOci_PU3LFsaNjSsgjbqGg,18676
 squad/ci/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 squad/ci/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 squad/ci/management/commands/create_tuxsuite_boot_tests.py,sha256=JvjNusebLX71eyz9d-kaeCyekYSpzc1eXoeIqWK9ygo,4045
@@ -82,7 +82,7 @@ squad/core/comparison.py,sha256=LR3-Unv0CTmakFCDzF_h8fm2peTJzkv79mQWNau1iwI,2442
 squad/core/data.py,sha256=2zw56v7iYRTUc7wlhuUNgwIIMmK2w84hi-amR9J7EPU,2236
 squad/core/failures.py,sha256=X6lJVghM2fOrd-RfuHeLlezW2pt7owDZ8eX-Kn_Qrt0,918
 squad/core/history.py,sha256=QRSIoDOw6R6vUWMtsPMknsHGM7FaCAeuCYqASCayHTk,3541
-squad/core/models.py,sha256=sXQmgPtl54IZT7rDmJEU3QK6JSPbi0hTUGRsjwL6PIo,60851
+squad/core/models.py,sha256=peMmwugkToA35bRZTdcJ_4TIpzT3Pj4GXCQ4bWeAcpY,60992
 squad/core/notification.py,sha256=rOpO6F63w7_5l9gQgWBBEk-MFBjp7x_hVzoVIVyDze0,10030
 squad/core/plugins.py,sha256=FLgyoXXKnPBYEf2MgHup9M017rHuADHivLhgzmx_cJE,6354
 squad/core/queries.py,sha256=78fhIJZWXIlDryewYAt96beK1VJad66Ufu8cg3dHh4w,7698
@@ -105,6 +105,7 @@ squad/core/management/commands/migrate_test_runs.py,sha256=RHV06tb4gWyv_q-ooC821
 squad/core/management/commands/populate_metric_build_and_environment.py,sha256=DJP9_YLRso0RiERBVsB0GP4-GaiRtJb0rAiUQDfFNQk,3166
 squad/core/management/commands/populate_test_build_and_environment.py,sha256=0yHClC0x_8LSZlvT6Ag0BnipC9Xk-U6lcIaCsqAGEWk,3146
 squad/core/management/commands/prepdump.py,sha256=WM58leVdJj45KhWPw3DGO7vwnNY70ReXrJRSIIzGXkI,518
+squad/core/management/commands/run_log_parser.py,sha256=SeksSD1cnbgl8oRsD3wu12p30_FMw090T6ouQyO4ZsI,1113
 squad/core/management/commands/send-email.py,sha256=wb1o5oKLDyH2ZonnQY-Jw28Y0Mu61OHWP8b1AQGKqbU,1120
 squad/core/management/commands/update_project_statuses.py,sha256=JleCesbVhYOSXr90ntH7s5u9Isknt7EnlX22VC6yI78,2089
 squad/core/management/commands/users.py,sha256=qIp87xRMfKWHymsAft5-gnYajm2mgaiHvVn7z86DCT8,9429
@@ -426,17 +427,18 @@ squad/plugins/__init__.py,sha256=9BSzy2jFIoDpWlhD7odPPrLdW4CC3btBhdFCvB651dM,152
 squad/plugins/example.py,sha256=BKpwd315lHRIuNXJPteibpwfnI6C5eXYHYdFYBtVmsI,89
 squad/plugins/gerrit.py,sha256=CqO2KnFQzu9utr_TQ-sGr1wg3ln0B-bS2-c0_i8T5-c,7009
 squad/plugins/github.py,sha256=pdtLZw_7xNuzkaFvY_zWi0f2rsMlalXjKm7sz0eADz4,2429
-squad/plugins/linux_log_parser.py,sha256=WrDbyfupEcP1-E4ke9wjHiddio8sD5BFuEtF4AH0aXA,3274
+squad/plugins/linux_log_parser.py,sha256=HQVreyZLBmLuv-K-MjlN43sQQSkcls4hkUsjJ9_5WfM,3472
+squad/plugins/linux_log_parser_build.py,sha256=KotAmKX9OCvnJqjaCudrMJu8CytXf_CYV3zqEKi2zNQ,10439
 squad/plugins/lib/__init__.py,sha256=jzazbAvp2_ibblAs0cKZrmo9aR2EL3hKLyRDE008r2I,40
-squad/plugins/lib/base_log_parser.py,sha256=OW6JkZ3PM5RiDkt9UZ7OFFpUIArCxFUaqovynzwBL1Y,6573
+squad/plugins/lib/base_log_parser.py,sha256=LiBCtx3dajSXITRTYxfgAmCkVS8KLI2527JKDa-D9ow,8430
 squad/run/__init__.py,sha256=ssE8GPAGFiK6V0WpZYowav6Zqsd63dfDMMYasNa1sQg,1410
 squad/run/__main__.py,sha256=DOl8JOi4Yg7DdtwnUeGqtYBJ6P2k-D2psAEuYOjWr8w,66
 squad/run/listener.py,sha256=jBeOQhPGb4EdIREB1QsCzYuumsfJ-TqJPd3nR-0m59g,200
 squad/run/scheduler.py,sha256=CDJG3q5C0GuQuxwlMOfWTSSJpDdwbR6rzpbJfuA0xuw,277
 squad/run/worker.py,sha256=jtML0h5qKDuSbpJ6_rpWP4MT_rsGA7a24AhwGxBquzk,594
-squad-1.91.dist-info/COPYING,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
-squad-1.91.dist-info/METADATA,sha256=U074123n6U3T9v0BY5GTaFCynFxFjL4HA7GEoLQjXX4,1278
-squad-1.91.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-squad-1.91.dist-info/entry_points.txt,sha256=J_jG3qnkoOHX4RFNGC0f83eJ4BSvK3pqLFkoF3HWfmA,195
-squad-1.91.dist-info/top_level.txt,sha256=_x9uqE1XppiiytmVTl_qNgpnXus6Gsef69HqfliE7WI,6
-squad-1.91.dist-info/RECORD,,
+squad-1.93.dist-info/COPYING,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
+squad-1.93.dist-info/METADATA,sha256=fQk8aD6kMXho1Q6eA5gllimHyMpDGRju3ql6qEo-3PM,1278
+squad-1.93.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+squad-1.93.dist-info/entry_points.txt,sha256=J_jG3qnkoOHX4RFNGC0f83eJ4BSvK3pqLFkoF3HWfmA,195
+squad-1.93.dist-info/top_level.txt,sha256=_x9uqE1XppiiytmVTl_qNgpnXus6Gsef69HqfliE7WI,6
+squad-1.93.dist-info/RECORD,,

{squad-1.91.dist-info → squad-1.93.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.44.0)
+Generator: bdist_wheel (0.45.1)
 Root-Is-Purelib: true
 Tag: py3-none-any