squad 1.89__tar.gz → 1.90__tar.gz

{squad-1.89 → squad-1.90}/CHANGELOG.md

@@ -1,3 +1,10 @@
+# 1.90
+
+This 1.90 release makes linux-log-parser more plugable, meaning that it could be
+moved away from SQUAD into its separate package.
+
+This release also improves performance when uploading testrun attachments.
+
 # 1.89
 
 This 1.89 release comes with quite several changes:

{squad-1.89 → squad-1.90}/MANIFEST.in

@@ -21,7 +21,6 @@ include .mailmap
 include .readthedocs.yml
 include .coveragerc
 include squad.svg
-include .github/workflows/*.yml
 include test/integration/plugins/tradefed-output-20220608105250.tar.xz
 
 include           squad/core/management/commands/import_data.rst
@@ -37,6 +36,7 @@ recursive-include .reuse/ *
 include babel.cfg
 recursive-include squad/ *.mo *.po *.pot
 
+exclude           .gitlab-ci.yml
 exclude           squad/local_settings.py
 exclude           squad/local_settings.py.off
 recursive-exclude squad/frontend/static/vendor/ *

squad-1.90/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.1
+Name: squad
+Version: 1.90
+Summary: Software Quality Dashboard
+Home-page: https://github.com/Linaro/squad
+Author: Antonio Terceiro
+Author-email: antonio.terceiro@linaro.org
+License: GPLv3+
+Platform: any
+Provides-Extra: postgres
+License-File: COPYING
+
+Software Quality Dashboard
+

{squad-1.89 → squad-1.90}/dev-docker

@@ -8,6 +8,20 @@ tmpdir="$basedir"/tmp
 datadir="$tmpdir"/data
 mkdir -p "$datadir"
 
+has_input=
+TTY="-i"
+if [ -t 1 ];
+then
+  TTY="-it"
+  has_input=yes
+fi
+
+user=squad
+if [ ! -z "${USER-}" ]
+then
+  user=${USER}
+fi
+
 dockerfile="$datadir"/../Dockerfile.dev
 (
   sed -e '1,/# Prepare the environment/!d' "$basedir"/Dockerfile
@@ -25,11 +39,16 @@ dockerfile="$datadir"/../Dockerfile.dev
 		nodejs \
 		ruby-foreman \
 		procps && \'
-  echo "    groupadd -g $(id -g) $(id -gn) && \\"
-  echo "    useradd -m -u $(id -u) -g $(id -g) -s /bin/bash ${USER}"
-  echo "RUN pip3 install --no-dependencies django-bootstrap3"
+  echo "    pip3 install --no-dependencies django-bootstrap3"
+
+  if [ ! -z "${has_input}" ]
+  then
+    echo "RUN groupadd -g $(id -g) $(id -gn) && \\"
+    echo "    useradd -m -u $(id -u) -g $(id -g) -s /bin/bash ${user}"
+    echo "USER ${user}"
+  fi
+
   echo "WORKDIR /app"
-  echo "USER ${USER}"
   echo 'CMD ["bash"]'
 ) > "$dockerfile"
 
@@ -44,12 +63,6 @@ for v in $HOME/.gitconfig $HOME/.config/git; do
   fi
 done
 
-TTY="-i"
-if [ -t 1 ];
-then
-  TTY="-it"
-fi
-
 PORT=${PORT:-8000}
 
 exec docker run \

{squad-1.89 → squad-1.90}/scripts/build

@@ -23,11 +23,8 @@ cleanup() {
         mv "$local_settings".off "$local_settings"
     fi
 }
-git ls-tree -r --name-only HEAD | grep -v '\.tar$' | sort > "$git"
+git ls-tree -r --name-only HEAD | grep -v '\(\.tar\|\.gitlab-ci.yml\)$' | sort > "$git"
 
 downloads=$(awk '{ if ($1 && $1 !~ /^#/) { print($1)} }' squad/frontend/static/download.conf | xargs printf '%s\|')
 tar taf dist/squad-${v}.tar.gz | cut -d / -f 2- | grep -v '\(/$\|^$\|PKG-INFO\|egg-info\|static\/\('$downloads'download.status\)\|.mo$\)' | sort > "$tar"
 diff -u "$tar" "$git"
-
-# Build docker container
-docker build -t squadproject/squad:$v .

{squad-1.89 → squad-1.90}/scripts/release

@@ -40,10 +40,13 @@ if [ "${TEST:-yes}" != "no" ]; then
     ./dev-docker ./manage.py test
 fi
 
-# Build squad pip package and docker container
+# Check squad pip package build
 ./scripts/build "v$v"
 rm -rf build/ dist/ *.egg-info/
 
+# Check docker container build
+docker build -t squadproject/squad:$v .
+
 # Commit changes and push the new tag
 git commit --message "New release: ${v}" CHANGELOG.md squad/version.py || true
 git tag -s -m "SQUAD release $v" "v$v"

{squad-1.89 → squad-1.90}/squad/api/views.py

@@ -105,7 +105,7 @@ def add_test_run(request, group_slug, project_slug, version, environment_slug):
     if 'attachment' in request.FILES:
         attachments = {}
         for f in request.FILES.getlist('attachment'):
-            attachments[f.name] = read_file_upload(f)
+            attachments[f.name] = f
         test_run_data['attachments'] = attachments
 
     receive = ReceiveTestRun(project)

{squad-1.89 → squad-1.90}/squad/core/management/commands/import_data.py

@@ -2,6 +2,7 @@ from glob import glob
 import os
 import re
 from django.core.management.base import BaseCommand
+from django.core.files import File
 
 
 from squad.core.models import Build
@@ -123,7 +124,7 @@ class Command(BaseCommand):
         for f in glob(os.path.join(directory, '*')):
             name = os.path.basename(f)
             if name not in ['metrics.json', 'metadata.json', 'tests.json']:
-                attachments[name] = open(f, 'rb').read()
+                attachments[name] = File(open(f, 'rb'))
 
         if not self.options['silent']:
             print("Importing test run: %s" % directory)

{squad-1.89 → squad-1.90}/squad/core/models.py

@@ -909,8 +909,8 @@ class Attachment(models.Model):
                 self.__data__ = b''
         return self.__data__
 
-    def save_file(self, filename, contents):
-        storage_save(self, self.storage, filename, contents)
+    def save_file(self, filename, file):
+        storage_save(self, self.storage, filename, file)
 
 
 class SuiteMetadata(models.Model):

{squad-1.89 → squad-1.90}/squad/core/tasks/__init__.py

@@ -172,9 +172,9 @@ class ReceiveTestRun(object):
         if log_file is not None:
             testrun.save_log_file(log_file)
 
-        for filename, data in attachments.items():
-            attachment = testrun.attachments.create(filename=filename, length=len(data))
-            attachment.save_file(filename, data)
+        for filename, file in attachments.items():
+            attachment = testrun.attachments.create(filename=filename, length=file.size)
+            attachment.save_file(filename, file)
 
         testrun.refresh_from_db()
 

{squad-1.89 → squad-1.90}/squad/core/utils.py

@@ -169,8 +169,11 @@ def log_deletion(request, object, message):
 
 
 def storage_save(obj, storage_field, filename, content):
-    content_bytes = content or ''
-    if type(content_bytes) is str:
-        content_bytes = content_bytes.encode()
     filename = '%s/%s/%s' % (obj.__class__.__name__.lower(), obj.pk, filename)
-    storage_field.save(filename, ContentFile(content_bytes))
+    if type(content) in [bytes, str]:
+        content_bytes = content or ''
+        if type(content_bytes) is str:
+            content_bytes = content_bytes.encode()
+        storage_field.save(filename, ContentFile(content_bytes))
+    else:
+        storage_field.save(filename, content)

squad-1.90/squad/plugins/lib/__init__.py

@@ -0,0 +1 @@
+from squad.plugins import Plugin # noqa

squad-1.90/squad/plugins/lib/base_log_parser.py

@@ -0,0 +1,152 @@
+import hashlib
+import re
+from collections import defaultdict
+
+from django.template.defaultfilters import slugify
+
+REGEX_NAME = 0
+REGEX_BODY = 1
+REGEX_EXTRACT_NAME = 2
+
+
+class BaseLogParser:
+    def compile_regexes(self, regexes):
+        combined = [r"(%s)" % r[REGEX_BODY] for r in regexes]
+        return re.compile(r"|".join(combined), re.S | re.M)
+
+    def remove_numbers_and_time(self, snippet):
+        without_numbers = re.sub(r"(0x[a-f0-9]+|[<\[][0-9a-f]+?[>\]]|\d+)", "", snippet)
+        without_time = re.sub(r"^\[[^\]]+\]", "", without_numbers)
+
+        return without_time
+
+    def create_name(self, snippet, compiled_regex=None):
+        matches = None
+        if compiled_regex:
+            matches = compiled_regex.findall(snippet)
+        if not matches:
+            # Only extract a name if we provide a regex to extract the name and
+            # there is a match
+            return None
+        snippet = matches[0]
+        without_numbers_and_time = self.remove_numbers_and_time(snippet)
+
+        # Limit the name length to 191 characters, since the max name length
+        # for SuiteMetadata in SQUAD is 256 characters. The SHA and "-" take 65
+        # characters: 256-65=191
+        return slugify(without_numbers_and_time)[:191]
+
+    def create_shasum(self, snippet):
+        sha = hashlib.sha256()
+        without_numbers_and_time = self.remove_numbers_and_time(snippet)
+        sha.update(without_numbers_and_time.encode())
+        return sha.hexdigest()
+
+    def create_name_log_dict(self, test_name, lines, test_regex=None):
+        """
+        Produce a dictionary with the test names as keys and the extracted logs
+        for that test name as values. There will be at least one test name per
+        regex. If there were any matches for a given regex, then a new test
+        will be generated using test_name + shasum.
+        """
+        # Run the REGEX_EXTRACT_NAME regex over the log lines to sort them by
+        # extracted name. If no name is extracted or the log parser did not
+        # have any output for a particular regex, just use the default name
+        # (for example "check-kernel-oops").
+        tests_without_shas_to_create = defaultdict(set)
+        tests_with_shas_to_create = defaultdict(set)
+
+        # If there are no lines, use the default name and create a passing
+        # test. For example "check-kernel-oops"
+        if not lines:
+            tests_without_shas_to_create[test_name] = []
+
+        # If there are lines, then create the tests for these.
+        for line in lines:
+            extracted_name = self.create_name(line, test_regex)
+            if extracted_name:
+                extended_test_name = f"{test_name}-{extracted_name}"
+            else:
+                extended_test_name = test_name
+            tests_without_shas_to_create[extended_test_name].add(line)
+
+        for name, test_lines in tests_without_shas_to_create.items():
+            # Some lines of the matched regex might be the same, and we don't want to create
+            # multiple tests like test1-sha1, test1-sha1, etc, so we'll create a set of sha1sums
+            # then create only new tests for unique sha's
+
+            for line in test_lines:
+                sha = self.create_shasum(line)
+                name_with_sha = f"{name}-{sha}"
+                tests_with_shas_to_create[name_with_sha].add(line)
+
+        return tests_without_shas_to_create, tests_with_shas_to_create
+
+    def create_squad_tests_from_name_log_dict(
+        self, suite, testrun, tests_without_shas_to_create, tests_with_shas_to_create
+    ):
+        # Import SuiteMetadata from SQUAD only when required so BaseLogParser
+        # does not require a SQUAD to work. This makes it easier to reuse this
+        # class outside of SQUAD for testing and developing log parser
+        # patterns.
+        from squad.core.models import SuiteMetadata
+
+        for name, lines in tests_without_shas_to_create.items():
+            metadata, _ = SuiteMetadata.objects.get_or_create(
+                suite=suite.slug, name=name, kind="test"
+            )
+            testrun.tests.create(
+                suite=suite,
+                result=(len(lines) == 0),
+                log="\n".join(lines),
+                metadata=metadata,
+                build=testrun.build,
+                environment=testrun.environment,
+            )
+        for name_with_sha, lines in tests_with_shas_to_create.items():
+            metadata, _ = SuiteMetadata.objects.get_or_create(
+                suite=suite.slug, name=name_with_sha, kind="test"
+            )
+            testrun.tests.create(
+                suite=suite,
+                result=False,
+                log="\n---\n".join(lines),
+                metadata=metadata,
+                build=testrun.build,
+                environment=testrun.environment,
+            )
+
+    def create_squad_tests(self, testrun, suite, test_name, lines, test_regex=None):
+        """
+        There will be at least one test per regex. If there were any match for
+        a given regex, then a new test will be generated using test_name +
+        shasum. This helps comparing kernel logs across different builds
+        """
+        tests_without_shas_to_create, tests_with_shas_to_create = (
+            self.create_name_log_dict(test_name, lines, test_regex)
+        )
+        self.create_squad_tests_from_name_log_dict(
+            suite,
+            testrun,
+            tests_without_shas_to_create,
+            tests_with_shas_to_create,
+        )
+
+    def join_matches(self, matches, regexes):
+        """
+        group regex in python are returned as a list of tuples which each
+        group match in one of the positions in the tuple. Example:
+        regex = r'(a)|(b)|(c)'
+        matches = [
+            ('match a', '', ''),
+            ('', 'match b', ''),
+            ('match a', '', ''),
+            ('', '', 'match c')
+        ]
+        """
+        snippets = {regex_id: [] for regex_id in range(len(regexes))}
+        for match in matches:
+            for regex_id in range(len(regexes)):
+                if len(match[regex_id]) > 0:
+                    snippets[regex_id].append(match[regex_id])
+        return snippets

squad-1.90/squad/plugins/linux_log_parser.py

@@ -0,0 +1,68 @@
+import logging
+import re
+from squad.plugins import Plugin as BasePlugin
+from squad.plugins.lib.base_log_parser import BaseLogParser, REGEX_NAME, REGEX_EXTRACT_NAME
+
+logger = logging.getLogger()
+
+MULTILINERS = [
+    ('check-kernel-exception', r'-+\[? cut here \]?-+.*?-+\[? end trace \w* \]?-+', r"\d][^\+\n]*"),
+    ('check-kernel-kasan', r'=+\n\[[\s\.\d]+\]\s+BUG: KASAN:.*?=+', r"BUG: KASAN:[^\+\n]*"),
+    ('check-kernel-kfence', r'=+\n\[[\s\.\d]+\]\s+BUG: KFENCE:.*?=+', r"BUG: KFENCE:[^\+\n]*"),
+]
+
+ONELINERS = [
+    ('check-kernel-oops', r'^[^\n]+Oops(?: -|:).*?$', r"Oops[^\+\n]*"),
+    ('check-kernel-fault', r'^[^\n]+Unhandled fault.*?$', r"Unhandled [^\+\n]*"),
+    ('check-kernel-warning', r'^[^\n]+WARNING:.*?$', r"WARNING: [^\+\n]*"),
+    ('check-kernel-bug', r'^[^\n]+(?: kernel BUG at|BUG:).*?$', r"BUG[^\+\n]*"),
+    ('check-kernel-invalid-opcode', r'^[^\n]+invalid opcode:.*?$', r"invalid opcode: [^\+\n]*"),
+    ('check-kernel-panic', r'Kernel panic - not syncing.*?$', r"Kernel [^\+\n]*"),
+]
+
+# Tip: broader regexes should come first
+REGEXES = MULTILINERS + ONELINERS
+
+
+class Plugin(BasePlugin, BaseLogParser):
+    def __cutoff_boot_log(self, log):
+        # Attempt to split the log in " login:"
+        logs = log.split(' login:', 1)
+
+        # 1 string means no split was done, consider all logs as test log
+        if len(logs) == 1:
+            return '', log
+
+        boot_log = logs[0]
+        test_log = logs[1]
+        return boot_log, test_log
+
+    def __kernel_msgs_only(self, log):
+        kernel_msgs = re.findall(r'(\[[ \d]+\.[ \d]+\] .*?)$', log, re.S | re.M)
+        return '\n'.join(kernel_msgs)
+
+    def postprocess_testrun(self, testrun):
+        if testrun.log_file is None:
+            return
+
+        boot_log, test_log = self.__cutoff_boot_log(testrun.log_file)
+        logs = {
+            'boot': boot_log,
+            'test': test_log,
+        }
+
+        for log_type, log in logs.items():
+            log = self.__kernel_msgs_only(log)
+            suite, _ = testrun.build.project.suites.get_or_create(slug=f'log-parser-{log_type}')
+
+            regex = self.compile_regexes(REGEXES)
+            matches = regex.findall(log)
+            snippets = self.join_matches(matches, REGEXES)
+
+            for regex_id in range(len(REGEXES)):
+                test_name = REGEXES[regex_id][REGEX_NAME]
+                regex_pattern = REGEXES[regex_id][REGEX_EXTRACT_NAME]
+                test_name_regex = None
+                if regex_pattern:
+                    test_name_regex = re.compile(regex_pattern, re.S | re.M)
+                self.create_squad_tests(testrun, suite, test_name, snippets[regex_id], test_name_regex)

{squad-1.89 → squad-1.90}/squad/settings.py

@@ -37,17 +37,21 @@ if not os.access(DATA_DIR, os.W_OK):
 # See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/
 
 # SECURITY WARNING: keep the secret key used in production secret!
-secret_key_file = os.getenv('SECRET_KEY_FILE', None)
-if secret_key_file is None:
-    secret_key_file = os.path.join(DATA_DIR, 'secret.dat')
-
-if not os.path.exists(secret_key_file):
-    from squad.core.utils import random_key
-    fd = os.open(secret_key_file, os.O_WRONLY | os.O_CREAT, 0o600)
-    with os.fdopen(fd, 'w') as f:
-        f.write(random_key(64))
-
-SECRET_KEY = open(secret_key_file).read()
+secret_key = os.getenv('SECRET_KEY', None)
+if secret_key:
+    SECRET_KEY = secret_key
+else:
+    secret_key_file = os.getenv('SECRET_KEY_FILE', None)
+    if secret_key_file is None:
+        secret_key_file = os.path.join(DATA_DIR, 'secret.dat')
+
+    if not os.path.exists(secret_key_file):
+        from squad.core.utils import random_key
+        fd = os.open(secret_key_file, os.O_WRONLY | os.O_CREAT, 0o600)
+        with os.fdopen(fd, 'w') as f:
+            f.write(random_key(64))
+
+    SECRET_KEY = open(secret_key_file).read()
 
 DEBUG = os.getenv('ENV') not in ['production', 'staging']
 

squad-1.90/squad/version.py

@@ -0,0 +1 @@
+__version__ = '1.90'

squad-1.90/squad.egg-info/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.1
+Name: squad
+Version: 1.90
+Summary: Software Quality Dashboard
+Home-page: https://github.com/Linaro/squad
+Author: Antonio Terceiro
+Author-email: antonio.terceiro@linaro.org
+License: GPLv3+
+Platform: any
+Provides-Extra: postgres
+License-File: COPYING
+
+Software Quality Dashboard
+

{squad-1.89 → squad-1.90}/squad.egg-info/SOURCES.txt

@@ -24,8 +24,6 @@ requirements.txt
 setup.cfg
 setup.py
 squad.svg
-.github/workflows/release.yml
-.github/workflows/test.yml
 .reuse/dep5
 LICENSES/GPL-3.0-or-later.txt
 LICENSES/MIT.txt
@@ -68,7 +66,6 @@ scripts/test-docker
 scripts/translate
 scripts/travis-lava
 scripts/update-translation-files
-scripts/upload
 scripts/community_connector/main.js
 scripts/community_connector/manifest.json
 scripts/testdata/gen-ci-jobs
@@ -512,6 +509,8 @@ squad/plugins/example.py
 squad/plugins/gerrit.py
 squad/plugins/github.py
 squad/plugins/linux_log_parser.py
+squad/plugins/lib/__init__.py
+squad/plugins/lib/base_log_parser.py
 squad/run/__init__.py
 squad/run/__main__.py
 squad/run/listener.py
@@ -616,6 +615,7 @@ test/plugins/__init__.py
 test/plugins/test_gerrit.py
 test/plugins/test_github.py
 test/plugins/test_linux_log_parser.py
+test/plugins/test_log_parser_base.py
 test/plugins/test_plugin.py
 test/plugins/linux_log_parser/kasan.log
 test/plugins/linux_log_parser/kernel_bug_and_invalid_opcode.log

{squad-1.89 → squad-1.90}/squad.egg-info/entry_points.txt

@@ -4,3 +4,4 @@ squad-admin = squad.manage:main
 squad-listener = squad.run.listener:main
 squad-scheduler = squad.run.scheduler:main
 squad-worker = squad.run.worker:main
+