squad 1.88__tar.gz → 1.89__tar.gz

{squad-1.88 → squad-1.89}/CHANGELOG.md

@@ -1,3 +1,23 @@
+# 1.89
+
+This 1.89 release comes with quite several changes:
+
+* api/rest.py: check if callback exception has messages attribute
+* ci:
+  * backend/lava.py:
+    * add sanity check in job_name
+    * fix cases for empty job_name in definition
+  * ci/tasks.py:
+    * check if job exists before fetching
+    * do not fetch jobs without job_id
+* core/models.py: prevend malformed yaml to be saved
+* core/callback.py: handle requests exceptions
+* core/history.py: check for empty projects
+* frontend/comparison.py: sanity check project slug
+* plugins/linux_log_parser:
+  * Add support for test name regex
+  * Move test name regex to higher level
+
 # 1.88
 
 This 1.88 release adds a neat feature that gives the user the exact log link

{squad-1.88 → squad-1.89}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: squad
-Version: 1.88
+Version: 1.89
 Summary: Software Quality Dashboard
 Home-page: https://github.com/Linaro/squad
 Author: Antonio Terceiro

{squad-1.88 → squad-1.89}/squad/api/rest.py

@@ -1190,7 +1190,13 @@ class BuildViewSet(NestedViewSetMixin, ModelViewSet):
                     raise ValidationError('url is required.')
                 return Response({'message': 'OK'}, status=status.HTTP_202_ACCEPTED)
             except (ValidationError, IntegrityError) as e:
-                return Response({'message': ', '.join(e.messages)}, status=status.HTTP_400_BAD_REQUEST)
+                message = ""
+                if hasattr(e, "messages"):
+                    message = ', '.join(e.messages)
+                else:
+                    message = str(e)
+
+                return Response({'message': message}, status=status.HTTP_400_BAD_REQUEST)
 
     @action(detail=True, methods=['get'], suffix='compare')
     def compare(self, request, pk=None):

{squad-1.88 → squad-1.89}/squad/ci/backend/lava.py

@@ -396,9 +396,10 @@ class Backend(BaseBackend):
 
     def __lava_job_name(self, definition):
         yaml_definition = yaml.safe_load(definition)
-        if 'job_name' in yaml_definition.keys():
+        if type(yaml_definition) is dict and 'job_name' in yaml_definition.keys():
+            job_name = yaml_definition['job_name']
             # only return first 255 characters
-            return yaml_definition['job_name'][:255]
+            return job_name[:255] if job_name else ''
         return None
 
     def __resubmit__(self, job_id):

{squad-1.88 → squad-1.89}/squad/ci/tasks.py

@@ -27,8 +27,12 @@ def poll(backend_id=None):
 @celery.task
 def fetch(job_id):
     logger.info("fetching %s" % job_id)
-    backend = TestJob.objects.get(pk=job_id).backend
-    backend.fetch(job_id)
+    try:
+        testjob = TestJob.objects.get(pk=job_id)
+        if testjob.job_id:
+            testjob.backend.fetch(testjob.id)
+    except TestJob.DoesNotExist:
+        return
 
 
 @celery.task

{squad-1.88 → squad-1.89}/squad/core/callback.py

@@ -56,12 +56,18 @@ def dispatch_callback(callback_object):
         else:
             args['data'] = json.loads(callback_object.payload)
 
-    request = getattr(requests, callback_object.method)
-    response = request(callback_object.url, **args)
+    try:
+        request = getattr(requests, callback_object.method)
+        response = request(callback_object.url, **args)
+        response_code = response.status_code
+        response_content = response.content
+    except requests.exceptions.RequestException as e:
+        response_code = None
+        response_content = str(e)
 
     if callback_object.record_response:
-        callback_object.response_code = response.status_code
-        callback_object.response_content = response.content
+        callback_object.response_code = response_code
+        callback_object.response_content = response_content
 
     callback_object.is_sent = True
     callback_object.save()

{squad-1.88 → squad-1.89}/squad/core/history.py

@@ -3,7 +3,7 @@ from django.core.paginator import Paginator
 
 from squad.core.queries import test_confidence
 from squad.core.utils import parse_name
-from squad.core.models import SuiteMetadata, KnownIssue, Environment
+from squad.core.models import SuiteMetadata, KnownIssue, Environment, Build
 
 
 class TestResult(object):
@@ -49,6 +49,9 @@ class TestHistory(object):
             self.number = page
             builds = self.paginator.page(page)
 
+        if len(builds) == 0:
+            raise Build.DoesNotExist
+
         self.top = builds[0]
 
         issues_by_env = {}

{squad-1.88 → squad-1.89}/squad/core/models.py

@@ -16,7 +16,7 @@ from django.db.models.query import prefetch_related_objects
 from django.contrib.auth.models import User, AnonymousUser, Group as auth_group
 from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation
 from django.contrib.contenttypes.models import ContentType
-from django.db.models.signals import post_save, pre_delete
+from django.db.models.signals import post_save, pre_delete, pre_save
 from django.dispatch import receiver
 
 from django.conf import settings
@@ -1428,9 +1428,13 @@ class ProjectStatus(models.Model, TestSummaryBase):
         return None
 
     def __get_yaml_field__(self, field_value):
-        if field_value is not None:
+        if field_value is None:
+            return {}
+
+        try:
             return yaml.load(field_value, Loader=yaml.Loader)
-        return {}
+        except yaml.YAMLError:
+            return {}
 
     def get_regressions(self):
         return self.__get_yaml_field__(self.regressions)
@@ -1467,6 +1471,14 @@ class ProjectStatus(models.Model, TestSummaryBase):
         return thresholds_exceeded
 
 
+@receiver(pre_save, sender=ProjectStatus)
+def validate_project_status(sender, instance, *args, **kwargs):
+    yaml_validator(instance.regressions)
+    yaml_validator(instance.fixes)
+    yaml_validator(instance.metric_regressions)
+    yaml_validator(instance.metric_fixes)
+
+
 class NotificationDelivery(models.Model):
 
     status = models.ForeignKey('ProjectStatus', related_name='deliveries', on_delete=models.CASCADE)

{squad-1.88 → squad-1.89}/squad/core/utils.py

@@ -67,7 +67,7 @@ def yaml_validator(value):
     if len(value) == 0:
         return
     try:
-        if not isinstance(yaml.safe_load(value), dict):
+        if not isinstance(yaml.load(value, Loader=yaml.Loader), dict):
             raise ValidationError("Dictionary object expected")
     except yaml.YAMLError as e:
         raise ValidationError(e)

{squad-1.88 → squad-1.89}/squad/frontend/comparison.py

@@ -3,6 +3,7 @@ from functools import reduce
 from django.shortcuts import render, get_object_or_404
 from django.core.paginator import Paginator
 from django.db.models import Q, Prefetch
+from django.http import HttpResponseNotFound
 
 from squad.core.models import Project, Group, Build
 from squad.core.comparison import TestComparison, MetricComparison
@@ -107,7 +108,11 @@ def compare_builds(request):
     comparison = None
     project = None
     if project_slug:
-        group_slug, project_slug = project_slug.split('/')
+        group_and_project = project_slug.split('/')
+        if len(group_and_project) != 2:
+            return HttpResponseNotFound()
+
+        group_slug, project_slug = group_and_project
         project = get_object_or_404(Project, group__slug=group_slug, slug=project_slug)
 
         baseline_build = request.GET.get('baseline')

{squad-1.88 → squad-1.89}/squad/frontend/tests.py

@@ -6,7 +6,7 @@ from django.shortcuts import render, get_object_or_404
 from django.http import Http404
 
 from squad.http import auth
-from squad.core.models import Test, Suite, SuiteMetadata, Environment
+from squad.core.models import Build, Test, Suite, SuiteMetadata, Environment
 from squad.core.history import TestHistory
 from squad.core.queries import test_confidence
 from squad.frontend.views import get_build
@@ -234,5 +234,5 @@ def test_history(request, group_slug, project_slug, build_version=None, testrun_
         history = TestHistory(project, full_test_name, top=top, page=page)
         context.update({"history": history})
         return render(request, 'squad/test_history.jinja2', context)
-    except (Suite.DoesNotExist, SuiteMetadata.DoesNotExist) as e:
+    except (Suite.DoesNotExist, SuiteMetadata.DoesNotExist, Build.DoesNotExist) as e:
         raise Http404(f"Test not found: {e}")

squad-1.89/squad/plugins/linux_log_parser.py

@@ -0,0 +1,183 @@
+import hashlib
+import logging
+import re
+from collections import defaultdict
+from squad.plugins import Plugin as BasePlugin
+from squad.core.models import SuiteMetadata
+from django.template.defaultfilters import slugify
+
+
+logger = logging.getLogger()
+
+REGEX_NAME = 0
+REGEX_BODY = 1
+REGEX_EXTRACT_NAME = 2
+
+MULTILINERS = [
+    ('check-kernel-exception', r'-+\[? cut here \]?-+.*?-+\[? end trace \w* \]?-+', r"\d][^\+\n]*"),
+    ('check-kernel-kasan', r'=+\n\[[\s\.\d]+\]\s+BUG: KASAN:.*?=+', r"BUG: KASAN:[^\+\n]*"),
+    ('check-kernel-kfence', r'=+\n\[[\s\.\d]+\]\s+BUG: KFENCE:.*?=+', r"BUG: KFENCE:[^\+\n]*"),
+]
+
+ONELINERS = [
+    ('check-kernel-oops', r'^[^\n]+Oops(?: -|:).*?$', r"Oops[^\+\n]*"),
+    ('check-kernel-fault', r'^[^\n]+Unhandled fault.*?$', r"Unhandled [^\+\n]*"),
+    ('check-kernel-warning', r'^[^\n]+WARNING:.*?$', r"WARNING: [^\+\n]*"),
+    ('check-kernel-bug', r'^[^\n]+(?: kernel BUG at|BUG:).*?$', r"BUG[^\+\n]*"),
+    ('check-kernel-invalid-opcode', r'^[^\n]+invalid opcode:.*?$', r"invalid opcode: [^\+\n]*"),
+    ('check-kernel-panic', r'Kernel panic - not syncing.*?$', r"Kernel [^\+\n]*"),
+]
+
+# Tip: broader regexes should come first
+REGEXES = MULTILINERS + ONELINERS
+
+
+class Plugin(BasePlugin):
+    def __compile_regexes(self, regexes):
+        combined = [r'(%s)' % r[REGEX_BODY] for r in regexes]
+        return re.compile(r'|'.join(combined), re.S | re.M)
+
+    def __cutoff_boot_log(self, log):
+        # Attempt to split the log in " login:"
+        logs = log.split(' login:', 1)
+
+        # 1 string means no split was done, consider all logs as test log
+        if len(logs) == 1:
+            return '', log
+
+        boot_log = logs[0]
+        test_log = logs[1]
+        return boot_log, test_log
+
+    def __kernel_msgs_only(self, log):
+        kernel_msgs = re.findall(r'(\[[ \d]+\.[ \d]+\] .*?)$', log, re.S | re.M)
+        return '\n'.join(kernel_msgs)
+
+    def __join_matches(self, matches, regexes):
+        """
+            group regex in python are returned as a list of tuples which each
+            group match in one of the positions in the tuple. Example:
+            regex = r'(a)|(b)|(c)'
+            matches = [
+                ('match a', '', ''),
+                ('', 'match b', ''),
+                ('match a', '', ''),
+                ('', '', 'match c')
+            ]
+        """
+        snippets = {regex_id: [] for regex_id in range(len(regexes))}
+        for match in matches:
+            for regex_id in range(len(regexes)):
+                if len(match[regex_id]) > 0:
+                    snippets[regex_id].append(match[regex_id])
+        return snippets
+
+    def __create_tests(self, testrun, suite, test_name, lines, test_regex=None):
+        """
+        There will be at least one test per regex. If there were any match for a given
+        regex, then a new test will be generated using test_name + shasum. This helps
+        comparing kernel logs accross different builds
+        """
+        # Run the REGEX_EXTRACT_NAME regex over the log lines to sort them by
+        # extracted name. If no name is extracted or the log parser did not
+        # have any output for a particular regex, just use the default name
+        # (for example "check-kernel-oops").
+        tests_to_create = defaultdict(set)
+        shas = defaultdict(set)
+
+        # If there are no lines, use the default name and create a passing
+        # test. For example "check-kernel-oops"
+        if not lines:
+            tests_to_create[test_name] = []
+
+        # If there are lines, then create the tests for these.
+        for line in lines:
+            extracted_name = self.__create_name(line, test_regex)
+            if extracted_name:
+                extended_test_name = f"{test_name}-{extracted_name}"
+            else:
+                extended_test_name = test_name
+            tests_to_create[extended_test_name].add(line)
+
+        for name, lines in tests_to_create.items():
+            metadata, _ = SuiteMetadata.objects.get_or_create(suite=suite.slug, name=name, kind='test')
+            testrun.tests.create(
+                suite=suite,
+                result=(len(lines) == 0),
+                log='\n'.join(lines),
+                metadata=metadata,
+                build=testrun.build,
+                environment=testrun.environment,
+            )
+
+            # Some lines of the matched regex might be the same, and we don't want to create
+            # multiple tests like test1-sha1, test1-sha1, etc, so we'll create a set of sha1sums
+            # then create only new tests for unique sha's
+
+            for line in lines:
+                sha = self.__create_shasum(line)
+                name_with_sha = f"{name}-{sha}"
+                shas[name_with_sha].add(line)
+
+        for name_with_sha, lines in shas.items():
+            metadata, _ = SuiteMetadata.objects.get_or_create(suite=suite.slug, name=name_with_sha, kind='test')
+            testrun.tests.create(
+                suite=suite,
+                result=False,
+                log='\n---\n'.join(lines),
+                metadata=metadata,
+                build=testrun.build,
+                environment=testrun.environment,
+            )
+
+    def __remove_numbers_and_time(self, snippet):
+        without_numbers = re.sub(r"(0x[a-f0-9]+|[<\[][0-9a-f]+?[>\]]|\d+)", "", snippet)
+        without_time = re.sub(r"^\[[^\]]+\]", "", without_numbers)
+
+        return without_time
+
+    def __create_name(self, snippet, regex=None):
+        matches = None
+        if regex:
+            matches = regex.findall(snippet)
+        if not matches:
+            return None
+        snippet = matches[0]
+        without_numbers_and_time = self.__remove_numbers_and_time(snippet)
+
+        # Limit the name length to 191 characters, since the max name length
+        # for SuiteMetadata in SQUAD is 256 characters. The SHA and "-" take 65
+        # characters: 256-65=191
+        return slugify(without_numbers_and_time)[:191]
+
+    def __create_shasum(self, snippet):
+        sha = hashlib.sha256()
+        without_numbers_and_time = self.__remove_numbers_and_time(snippet)
+        sha.update(without_numbers_and_time.encode())
+        return sha.hexdigest()
+
+    def postprocess_testrun(self, testrun):
+        if testrun.log_file is None:
+            return
+
+        boot_log, test_log = self.__cutoff_boot_log(testrun.log_file)
+        logs = {
+            'boot': boot_log,
+            'test': test_log,
+        }
+
+        for log_type, log in logs.items():
+            log = self.__kernel_msgs_only(log)
+            suite, _ = testrun.build.project.suites.get_or_create(slug=f'log-parser-{log_type}')
+
+            regex = self.__compile_regexes(REGEXES)
+            matches = regex.findall(log)
+            snippets = self.__join_matches(matches, REGEXES)
+
+            for regex_id in range(len(REGEXES)):
+                test_name = REGEXES[regex_id][REGEX_NAME]
+                regex_pattern = REGEXES[regex_id][REGEX_EXTRACT_NAME]
+                test_name_regex = None
+                if regex_pattern:
+                    test_name_regex = re.compile(regex_pattern, re.S | re.M)
+                self.__create_tests(testrun, suite, test_name, snippets[regex_id], test_name_regex)

squad-1.89/squad/version.py

@@ -0,0 +1 @@
+__version__ = '1.89'

{squad-1.88 → squad-1.89}/squad.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: squad
-Version: 1.88
+Version: 1.89
 Summary: Software Quality Dashboard
 Home-page: https://github.com/Linaro/squad
 Author: Antonio Terceiro

{squad-1.88 → squad-1.89}/test/ci/backend/test_lava.py

@@ -448,6 +448,16 @@ class LavaTest(TestCase):
         check = lava.check_job_definition(definition)
         self.assertIn('found unexpected end of stream', check)
 
+    def test_lava_job_name(self):
+        lava = LAVABackend(None)
+        self.assertIsNone(lava._Backend__lava_job_name('no name:'))
+        self.assertIsNone(lava._Backend__lava_job_name('- list'))
+        self.assertIsNone(lava._Backend__lava_job_name('string'))
+        self.assertEqual('', lava._Backend__lava_job_name('job_name:'))
+        self.assertEqual('job-name', lava._Backend__lava_job_name('job_name: job-name'))
+        truncated_name = lava._Backend__lava_job_name('job_name: ' + ('a' * 300))
+        self.assertEqual(255, len(truncated_name))
+
     @patch("requests.post", side_effect=requests.exceptions.Timeout)
     def test_submit_timeout(self, post):
         test_definition = "foo: 1\njob_name: bar"

{squad-1.88 → squad-1.89}/test/ci/test_tasks.py

@@ -122,6 +122,21 @@ class FetchTest(TestCase):
         self.test_job.refresh_from_db()
         self.assertEqual(attemps + 1, self.test_job.fetch_attempts)
 
+    @patch('squad.ci.models.Backend.fetch')
+    def test_fetch_no_job_id(self, fetch_method):
+        testjob = models.TestJob.objects.create(
+            backend=self.test_job.backend,
+            target=self.test_job.target,
+            target_build=self.test_job.target_build,
+        )
+        fetch.apply(args=[testjob.id])
+        fetch_method.assert_not_called()
+
+    @patch('squad.ci.models.Backend.fetch')
+    def test_fetch_deleted_job(self, fetch_method):
+        fetch.apply(args=[99999999999])
+        fetch_method.assert_not_called()
+
 
 class FetchTestRaceCondition(TransactionTestCase):
 
@@ -134,21 +149,23 @@ class FetchTestRaceCondition(TransactionTestCase):
             backend=backend,
             target=project,
             target_build=build,
+            job_id='test',
         )
 
     def mock_backend_fetch(test_job):
-        time.sleep(1)
+        time.sleep(0.5)
         status = ''
         completed = True
         metadata = {}
-        tests = []
-        metrics = []
+        tests = {}
+        metrics = {}
         logs = ''
         return status, completed, metadata, tests, metrics, logs
 
     @tag('skip_sqlite')
+    @patch('squad.ci.backend.null.Backend.job_url', return_value='http://url')
     @patch('squad.ci.backend.null.Backend.fetch', side_effect=mock_backend_fetch)
-    def test_race_condition_on_fetch(self, fetch_method):
+    def test_race_condition_on_fetch(self, fetch_method, job_url_method):
 
         def thread(testjob_id):
             fetch(testjob_id)

{squad-1.88 → squad-1.89}/test/core/test_callback.py

@@ -1,4 +1,5 @@
 import json
+import requests
 
 
 from django.test import TestCase
@@ -98,6 +99,19 @@ class CallbackTest(TestCase):
         callback.dispatch()
         self.assertFalse(requests_post.called)
 
+    @patch('requests.post')
+    def test_build_callback_catch_exceptions_on_dispatch(self, requests_post):
+        requests_post.side_effect = requests.exceptions.ConnectionError("bad connection")
+
+        url = 'http://callback-target.com'
+
+        callback = self.build.callbacks.create(url=url, event=self.event, record_response=True)
+        callback.dispatch()
+
+        self.assertTrue(requests_post.called)
+        self.assertTrue(callback.is_sent)
+        self.assertEqual("bad connection", callback.response_content)
+
     @patch('requests.post')
     def test_build_callback_gets_deleted_on_build_deletion(self, requests_post):
         url = 'http://callback-target.com'

{squad-1.88 → squad-1.89}/test/core/test_history.py

@@ -113,3 +113,10 @@ class TestHistoryTest(TestCase):
         history = TestHistory(self.project1, test_name)
 
         self.assertIsNotNone(history.results)
+
+    def test_no_build(self):
+        empty_project = self.group.projects.create(slug='empty-project')
+        test_name = 'fake_test'
+
+        with self.assertRaises(models.Build.DoesNotExist):
+            TestHistory(empty_project, test_name)

{squad-1.88 → squad-1.89}/test/core/test_project_status.py

@@ -1,3 +1,4 @@
+from django.core.exceptions import ValidationError
 from django.utils import timezone
 from django.test import TestCase
 from dateutil.relativedelta import relativedelta
@@ -436,3 +437,19 @@ class ProjectStatusTest(TestCase):
 
         self.assertEqual(build2.status.baseline, build1)
         self.assertEqual(build3.status.baseline, build1)
+
+    def test_save_malformed_yaml(self):
+        build = self.project.builds.create(version="bad-yaml")
+        build.status.fixes = """!!python/object/apply:collections.OrderedDict
+            - - - env
+            - [suite/test"""
+        with self.assertRaises(ValidationError):
+            build.status.save()
+
+    def test_get_malformed_yaml(self):
+        # For backwards compatibility, records with bad yaml already saved
+        build = self.project.builds.create(version="bad-yaml")
+        build.status.fixes = """!!python/object/apply:collections.OrderedDict
+            - - - env
+            - [suite/test"""
+        self.assertEqual(0, len(build.status.get_fixes()))

{squad-1.88 → squad-1.89}/test/core/test_update_project_statuses.py

@@ -27,13 +27,13 @@ class UpdateStatusesTest(TestCase):
         build1.save()
 
         status1 = ProjectStatus.objects.first()
-        status1.fixes = "fixes1"
+        status1.fixes = "fixes:\n- fix1"
         status1.finished = True
         status1.save()
 
         self.create_build('2')
         status2 = ProjectStatus.objects.last()
-        status2.fixes = "fixes2"
+        status2.fixes = "fixes:\n- fix2"
         status2.finished = True
         status2.save()
 
@@ -42,5 +42,5 @@ class UpdateStatusesTest(TestCase):
         status1.refresh_from_db()
         status2.refresh_from_db()
 
-        self.assertEqual(status1.fixes, "fixes1")
-        self.assertNotEqual(status2.fixes, "fixes2")
+        self.assertEqual(status1.get_fixes(), {"fixes": ["fix1"]})
+        self.assertEqual(status2.get_fixes(), {})

{squad-1.88 → squad-1.89}/test/frontend/test_comparison.py

@@ -78,3 +78,11 @@ class ProjectComparisonTest(TestCase):
         self.assertIn('otherenv', str(response.content))
         self.assertIn('pass', str(response.content))
         self.assertIn('fail', str(response.content))
+
+
+class BuildComparisonTest(TestCase):
+
+    def test_comparison_malformed_project_slug(self):
+        url = '/_/comparebuilds/?project=something-bad'
+        response = self.client.get(url)
+        self.assertEqual(404, response.status_code)