squad 1.89__tar.gz → 1.91__tar.gz

{squad-1.89 → squad-1.91}/CHANGELOG.md

@@ -1,3 +1,50 @@
+# 1.91
+
+This 1.91 release brings a few changes worth pointing out separately:
+
+* Tuxsuite backend: add support for changing environment name if "test_name" is present. Artificial boot tests
+when jobs fail or get canceled. Also checks if the testjob's project contains a token allowing authenticated
+urls in Tuxsuite.
+
+* Log parser: Katie is putting a lot of work in improving linux-log-parser plugin. The idea is that it will
+become its own project in the future
+
+* Add support for performance tracking when supported (Sentry) 
+
+* Support for prometheus-like metrics output for supporting environments (AMQ)
+
+Full set of changes going in:
+
+* api/ci: check job id before accepting test job
+* api/prometheus.py: add initial prometheus metrics endpoint
+* ci/tuxsuite.py:
+  * add support for fetching authenticated tuxsuite urls
+  * change environment name if test_name is available
+  * create artificial boot test
+  * handle canceled tests
+* ci: lava: fix login test name
+* frontend/project.jinja2: add project description to project's home page
+* frontend: urls: point test to details page
+* plugins/linux_log_parser:
+  * Add a new multiline regex for KCSAN
+  * Add regex for capturing multiline kernel panic
+  * Ensure .* regex doesn't capture too much
+  * Extract reused regex components
+  * No longer create tests for passes
+  * Remove unhelpful information from test names
+  * Add support for PID after timestamp
+  * Add support for multiline oops message
+  * Improve log parser patterns
+* setup: change package to psycopg2-binary
+
+
+# 1.90
+
+This 1.90 release makes linux-log-parser more plugable, meaning that it could be
+moved away from SQUAD into its separate package.
+
+This release also improves performance when uploading testrun attachments.
+
 # 1.89
 
 This 1.89 release comes with quite several changes:

{squad-1.89 → squad-1.91}/MANIFEST.in

@@ -21,7 +21,6 @@ include .mailmap
 include .readthedocs.yml
 include .coveragerc
 include squad.svg
-include .github/workflows/*.yml
 include test/integration/plugins/tradefed-output-20220608105250.tar.xz
 
 include           squad/core/management/commands/import_data.rst
@@ -37,6 +36,7 @@ recursive-include .reuse/ *
 include babel.cfg
 recursive-include squad/ *.mo *.po *.pot
 
+exclude           .gitlab-ci.yml
 exclude           squad/local_settings.py
 exclude           squad/local_settings.py.off
 recursive-exclude squad/frontend/static/vendor/ *

squad-1.91/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.1
+Name: squad
+Version: 1.91
+Summary: Software Quality Dashboard
+Home-page: https://github.com/Linaro/squad
+Author: Antonio Terceiro
+Author-email: antonio.terceiro@linaro.org
+License: GPLv3+
+Platform: any
+Provides-Extra: postgres
+License-File: COPYING
+
+Software Quality Dashboard
+

{squad-1.89 → squad-1.91}/dev-docker

@@ -8,6 +8,20 @@ tmpdir="$basedir"/tmp
 datadir="$tmpdir"/data
 mkdir -p "$datadir"
 
+has_input=
+TTY="-i"
+if [ -t 1 ];
+then
+  TTY="-it"
+  has_input=yes
+fi
+
+user=squad
+if [ ! -z "${USER-}" ]
+then
+  user=${USER}
+fi
+
 dockerfile="$datadir"/../Dockerfile.dev
 (
   sed -e '1,/# Prepare the environment/!d' "$basedir"/Dockerfile
@@ -25,11 +39,16 @@ dockerfile="$datadir"/../Dockerfile.dev
 		nodejs \
 		ruby-foreman \
 		procps && \'
-  echo "    groupadd -g $(id -g) $(id -gn) && \\"
-  echo "    useradd -m -u $(id -u) -g $(id -g) -s /bin/bash ${USER}"
-  echo "RUN pip3 install --no-dependencies django-bootstrap3"
+  echo "    pip3 install --no-dependencies django-bootstrap3"
+
+  if [ ! -z "${has_input}" ]
+  then
+    echo "RUN groupadd -g $(id -g) $(id -gn) && \\"
+    echo "    useradd -m -u $(id -u) -g $(id -g) -s /bin/bash ${user}"
+    echo "USER ${user}"
+  fi
+
   echo "WORKDIR /app"
-  echo "USER ${USER}"
   echo 'CMD ["bash"]'
 ) > "$dockerfile"
 
@@ -44,12 +63,6 @@ for v in $HOME/.gitconfig $HOME/.config/git; do
   fi
 done
 
-TTY="-i"
-if [ -t 1 ];
-then
-  TTY="-it"
-fi
-
 PORT=${PORT:-8000}
 
 exec docker run \

{squad-1.89 → squad-1.91}/doc/install.rst

@@ -214,6 +214,13 @@ __ https://docs.djangoproject.com/en/1.11/topics/email/
   it's recommended to disable sending admin notifications by setting
   ``SQUAD_SEND_ADMIN_ERROR_EMAIL = False``.
 
+* ``SENTRY_TRACES_SAMPLE_RATE``: Specifies the rate at which traces
+  (for performance monitoring) are sampled by Sentry. This value must be a
+  float between 0.0 and 1.0, where 1.0 represents 100% of traces being
+  captured. If not set, it defaults to 0. This option is used to configure
+  Sentry's performance monitoring, allowing you to control how much tracing
+  data is sent to Sentry. It works alongside the SENTRY_DSN setting.
+
 * ``SQUAD_STATIC_DIR``: Directory where SQUAD will find it's preprocessed
   static assets. This usually does not need to be set manually, and exists
   mostly for use in the Docker image.

{squad-1.89 → squad-1.91}/scripts/build

@@ -23,11 +23,8 @@ cleanup() {
         mv "$local_settings".off "$local_settings"
     fi
 }
-git ls-tree -r --name-only HEAD | grep -v '\.tar$' | sort > "$git"
+git ls-tree -r --name-only HEAD | grep -v '\(\.tar\|\.gitlab-ci.yml\)$' | sort > "$git"
 
 downloads=$(awk '{ if ($1 && $1 !~ /^#/) { print($1)} }' squad/frontend/static/download.conf | xargs printf '%s\|')
 tar taf dist/squad-${v}.tar.gz | cut -d / -f 2- | grep -v '\(/$\|^$\|PKG-INFO\|egg-info\|static\/\('$downloads'download.status\)\|.mo$\)' | sort > "$tar"
 diff -u "$tar" "$git"
-
-# Build docker container
-docker build -t squadproject/squad:$v .

{squad-1.89 → squad-1.91}/scripts/release

@@ -40,10 +40,13 @@ if [ "${TEST:-yes}" != "no" ]; then
     ./dev-docker ./manage.py test
 fi
 
-# Build squad pip package and docker container
+# Check squad pip package build
 ./scripts/build "v$v"
 rm -rf build/ dist/ *.egg-info/
 
+# Check docker container build
+docker build -t squadproject/squad:$v .
+
 # Commit changes and push the new tag
 git commit --message "New release: ${v}" CHANGELOG.md squad/version.py || true
 git tag -s -m "SQUAD release $v" "v$v"

{squad-1.89 → squad-1.91}/setup.py

@@ -18,7 +18,7 @@ if os.getenv('REQ_IGNORE_VERSIONS'):
     requirements = [req.split('>=')[0] for req in requirements]
 
 extras_require = {
-    'postgres': 'psycopg2',
+    'postgres': 'psycopg2-binary',
 }
 
 

{squad-1.89 → squad-1.91}/squad/api/ci.py

@@ -73,15 +73,26 @@ def submit_job(request, group_slug, project_slug, version, environment_slug):
 @csrf_exempt
 @auth_privileged
 def watch_job(request, group_slug, project_slug, version, environment_slug):
+
+    # testjob_id points to the backend's test job
+    testjob_id = request.POST.get('testjob_id', None)
+    if testjob_id is None:
+        return HttpResponseBadRequest("testjob_id is required")
+
     backend_name = request.POST.get('backend')
     if backend_name is None:
         return HttpResponseBadRequest("backend field is required")
+
     backend = None
     try:
         backend = Backend.objects.get(name=request.POST.get('backend'))
     except Backend.DoesNotExist:
         return HttpResponseBadRequest("requested backend does not exist")
 
+    check = backend.get_implementation().check_job_id(testjob_id)
+    if check is not True:
+        return HttpResponseBadRequest(check)
+
     # project has to exist or request will result with 400
     project = request.project
     if backend is None or project is None:
@@ -90,12 +101,6 @@ def watch_job(request, group_slug, project_slug, version, environment_slug):
     # create Build object
     build, _ = project.builds.get_or_create(version=version)
 
-    # testjob_id points to the backend's test job
-    testjob_id = request.POST.get('testjob_id', None)
-
-    if testjob_id is None:
-        return HttpResponseBadRequest("testjob_id is required")
-
     # create TestJob object
     test_job = TestJob(
         backend=backend,

squad-1.91/squad/api/prometheus.py

@@ -0,0 +1,52 @@
+import re
+import requests
+import celery
+
+from django.conf import settings
+from django.http import HttpResponse, HttpResponseForbidden
+from django.views.decorators.csrf import csrf_exempt
+from django.views.decorators.http import require_http_methods
+
+from squad.http import auth_user_from_request
+
+
+@csrf_exempt
+@require_http_methods(['GET'])
+def metrics(request):
+    user = auth_user_from_request(request, request.user)
+    if not user.is_authenticated:
+        return HttpResponseForbidden()
+
+    output = ''
+    available_queues = None
+
+    active_queues = celery.current_app.control.inspect().active_queues()
+    if active_queues is not None:
+        active_workers = set()
+        available_queues = set()
+        for worker_name, queues in active_queues.items():
+            active_workers.add(worker_name)
+            available_queues |= set([q['name'] for q in queues])
+
+        output += '# TYPE workers_count counter\n'
+        output += f'workers_count {len(active_workers)}\n'
+
+    # TODO: check how to get metrics for non-RabbitMQ brokers
+    if settings.CELERY_BROKER_URL:
+        rabbitmq_url = settings.CELERY_BROKER_URL.replace('amqps://', 'https://').replace('amqp://', 'http://')
+        rabbitmq_url = re.sub(r':\d+$', '', rabbitmq_url)
+        rabbitmq_url += '/api/queues'
+
+        response = requests.get(rabbitmq_url)
+        queues = response.json()
+        available_queues = {r["queue"] for r in settings.CELERY_TASK_ROUTES.values()}
+
+        for queue in queues:
+            if queue['name'] in available_queues:
+                metric_name = f'queue_{queue["name"]}_length'
+                length = queue['messages_ready']
+
+                output += f'\n# TYPE {metric_name} counter'
+                output += f'\n{metric_name} {length}'
+
+    return HttpResponse(output, status=200, content_type="text/plain;")

{squad-1.89 → squad-1.91}/squad/api/urls.py

@@ -5,6 +5,7 @@ from rest_framework.schemas import get_schema_view
 from . import views
 from . import data
 from . import ci
+from . import prometheus
 from . import rest
 
 
@@ -26,4 +27,5 @@ urlpatterns = [
     url(r'^resubmit/([0-9]+)', ci.resubmit_job),
     url(r'^forceresubmit/([0-9]+)', ci.force_resubmit_job),
     url(r'^version/', views.version),
+    url(r'^prometheus/', prometheus.metrics),
 ]

{squad-1.89 → squad-1.91}/squad/api/views.py

@@ -105,7 +105,7 @@ def add_test_run(request, group_slug, project_slug, version, environment_slug):
     if 'attachment' in request.FILES:
         attachments = {}
         for f in request.FILES.getlist('attachment'):
-            attachments[f.name] = read_file_upload(f)
+            attachments[f.name] = f
         test_run_data['attachments'] = attachments
 
     receive = ReceiveTestRun(project)

{squad-1.89 → squad-1.91}/squad/ci/backend/fake.py

@@ -72,6 +72,9 @@ class Backend(object):
     def check_job_definition(self, definition):
         return True
 
+    def check_job_id(self, job_id):
+        return True
+
     def get_job_definition(self, test_job):
         return "sample job definition"
 

{squad-1.89 → squad-1.91}/squad/ci/backend/lava.py

@@ -683,7 +683,7 @@ class Backend(BaseBackend):
                         if clone_measurements_to_tests:
                             res_value = result['result']
                             results.update({res_name: res_value})
-                elif result['name'] == 'auto-login-action' and handle_lava_boot:
+                elif 'login-action' in result['name'] and handle_lava_boot:
                     # add artificial 'boot' test result for each test job
                     # by default the boot test is named after the device_type
                     boot = "boot-%s" % test_job.name
@@ -785,6 +785,11 @@ class Backend(BaseBackend):
         except yaml.YAMLError as e:
             return str(e)
 
+    def check_job_id(self, job_id):
+        if re.match(r"^\d+$", str(job_id)) is not None:
+            return True
+        return "LAVA job id should be an integer"
+
     def get_job_definition(self, job_id):
         if self.use_xml_rpc:
             return self.proxy.scheduler.jobs.definition(job_id)

{squad-1.89 → squad-1.91}/squad/ci/backend/null.py

@@ -143,6 +143,12 @@ class Backend:
         """
         raise NotImplementedError
 
+    def check_job_id(self, job_id):
+        """
+        Returns True if job id matches what the backend expect, else returns the error message
+        """
+        raise NotImplementedError
+
     def format_message(self, msg):
         if self.data and hasattr(self.data, "name"):
             return self.data.name + ': ' + msg

{squad-1.89 → squad-1.91}/squad/ci/backend/tuxsuite.py

@@ -120,6 +120,13 @@ class Backend(BaseBackend):
         # The regex below is supposed to find only one match
         return matches[0]
 
+    def check_job_id(self, job_id):
+        try:
+            self.parse_job_id(job_id)
+            return True
+        except FetchIssue as e:
+            return str(e)
+
     def generate_job_id(self, result_type, result):
         """
             The job id for TuxSuite results is generated using 3 pieces of info:
@@ -144,7 +151,11 @@ class Backend(BaseBackend):
         url = reduce(urljoin, urlbits)
 
         try:
-            response = Backend.get_session().request("GET", url)
+            headers = {}
+            if hasattr(self, 'auth_token') and self.auth_token is not None:
+                headers = {'Authorization': self.auth_token}
+
+            response = Backend.get_session().request("GET", url, headers=headers)
         except Exception as e:
             raise TemporaryFetchIssue(f"Can't retrieve from {url}: {e}")
 
@@ -224,6 +235,11 @@ class Backend(BaseBackend):
         if 'toolchain' in build_metadata_keys and 'kconfig' in build_metadata_keys and metadata['build_name'] in [None, '']:
             metadata['build_name'] = self.generate_test_name(build_metadata)
 
+    def add_skip_boot_test(self, tests, metadata):
+        # Create an artificial boot test and mark it as skip
+        boot_test_name = 'boot/' + (metadata.get('build_name') or 'boot')
+        tests[boot_test_name] = None
+
     def parse_build_results(self, test_job, job_url, results, settings):
         required_keys = ['build_status', 'warnings_count', 'download_url', 'retry']
         self.__check_required_keys__(required_keys, results)
@@ -318,6 +334,11 @@ class Backend(BaseBackend):
         metadata_keys = settings.get('TEST_METADATA_KEYS', [])
         metadata = {k: results.get(k) for k in metadata_keys}
 
+        # Change environment name
+        if 'test_name' in results and results.get('test_name') is not None:
+            test_job.environment = results.get('test_name')
+            test_job.save()
+
         # Add extra metadata from metadata file if it exists
         self.update_metadata_from_file(results=results, metadata=metadata)
 
@@ -340,6 +361,8 @@ class Backend(BaseBackend):
             else:
                 test_job.failure = 'sanity test failed'
 
+            self.add_skip_boot_test(tests, metadata)
+
             return status, completed, metadata, tests, metrics, logs
 
         # Fetch results even if the job fails, but has results
@@ -348,8 +371,14 @@ class Backend(BaseBackend):
 
         elif results['result'] == 'error':
             test_job.failure = 'tuxsuite infrastructure error'
+            self.add_skip_boot_test(tests, metadata)
             return 'Incomplete', completed, metadata, tests, metrics, logs
 
+        elif results['result'] == 'canceled':
+            test_job.failure = 'tuxsuite job canceled'
+            self.add_skip_boot_test(tests, metadata)
+            return 'Canceled', completed, metadata, tests, metrics, logs
+
         # If boot result is unkown, a retry is needed, otherwise, it either passed or failed
         if 'unknown' == results['results']['boot']:
             return None
@@ -384,6 +413,10 @@ class Backend(BaseBackend):
 
     def fetch(self, test_job):
         url = self.job_url(test_job)
+
+        settings = self.__resolve_settings__(test_job)
+        self.auth_token = settings.get('TUXSUITE_TOKEN', None)
+
         if test_job.input:
             results = self.fetch_from_results_input(test_job)
             test_job.input = None
@@ -393,11 +426,12 @@ class Backend(BaseBackend):
         if results.get('state') != 'finished':
             return None
 
-        settings = self.__resolve_settings__(test_job)
-
         result_type = self.parse_job_id(test_job.job_id)[0]
         parse_results = getattr(self, f'parse_{result_type.lower()}_results')
-        return parse_results(test_job, url, results, settings)
+        parsed = parse_results(test_job, url, results, settings)
+
+        self.auth_token = None
+        return parsed
 
     def job_url(self, test_job):
         result_type, tux_project, tux_uid = self.parse_job_id(test_job.job_id)

{squad-1.89 → squad-1.91}/squad/core/management/commands/import_data.py

@@ -2,6 +2,7 @@ from glob import glob
 import os
 import re
 from django.core.management.base import BaseCommand
+from django.core.files import File
 
 
 from squad.core.models import Build
@@ -123,7 +124,7 @@ class Command(BaseCommand):
         for f in glob(os.path.join(directory, '*')):
             name = os.path.basename(f)
             if name not in ['metrics.json', 'metadata.json', 'tests.json']:
-                attachments[name] = open(f, 'rb').read()
+                attachments[name] = File(open(f, 'rb'))
 
         if not self.options['silent']:
             print("Importing test run: %s" % directory)

{squad-1.89 → squad-1.91}/squad/core/models.py

@@ -909,8 +909,8 @@ class Attachment(models.Model):
                 self.__data__ = b''
         return self.__data__
 
-    def save_file(self, filename, contents):
-        storage_save(self, self.storage, filename, contents)
+    def save_file(self, filename, file):
+        storage_save(self, self.storage, filename, file)
 
 
 class SuiteMetadata(models.Model):

{squad-1.89 → squad-1.91}/squad/core/tasks/__init__.py

@@ -172,9 +172,9 @@ class ReceiveTestRun(object):
         if log_file is not None:
             testrun.save_log_file(log_file)
 
-        for filename, data in attachments.items():
-            attachment = testrun.attachments.create(filename=filename, length=len(data))
-            attachment.save_file(filename, data)
+        for filename, file in attachments.items():
+            attachment = testrun.attachments.create(filename=filename, length=file.size)
+            attachment.save_file(filename, file)
 
         testrun.refresh_from_db()
 

{squad-1.89 → squad-1.91}/squad/core/utils.py

@@ -169,8 +169,11 @@ def log_deletion(request, object, message):
 
 
 def storage_save(obj, storage_field, filename, content):
-    content_bytes = content or ''
-    if type(content_bytes) is str:
-        content_bytes = content_bytes.encode()
     filename = '%s/%s/%s' % (obj.__class__.__name__.lower(), obj.pk, filename)
-    storage_field.save(filename, ContentFile(content_bytes))
+    if type(content) in [bytes, str]:
+        content_bytes = content or ''
+        if type(content_bytes) is str:
+            content_bytes = content_bytes.encode()
+        storage_field.save(filename, ContentFile(content_bytes))
+    else:
+        storage_field.save(filename, content)

{squad-1.89 → squad-1.91}/squad/frontend/templates/squad/project.jinja2

@@ -6,6 +6,14 @@
 
 {% include "squad/project-nav.jinja2" %}
 
+
+{% if project.description%}
+<h2>{{ _('Description') }}</h2>
+<div class='description-{{project.id}}'>
+    {{project.description}}
+</div>
+{% endif %}
+
 {% if last_build %}
 <div>
 <h2>

{squad-1.89 → squad-1.91}/squad/frontend/urls.py

@@ -52,6 +52,7 @@ urlpatterns = [
     url(r'^(%s)/(%s)/build/([^/]+)/attachments/testrun/([^/]+)/([^/]+)$' % group_and_project, views.build_attachment, name='build_attachments'),
     url(r'^(%s)/(%s)/build/([^/]+)/testrun/([^/]+)/suite/([^/]+)/tests/$' % group_and_project, views.test_run_suite_tests, name='testrun_suite_tests'),
     url(r'^(%s)/(%s)/build/([^/]+)/testrun/([^/]+)/suite/([^/]+)/test/([^/]+)/history/$' % group_and_project, tests.test_history, name='test_history'),
+    url(r'^(%s)/(%s)/build/([^/]+)/testrun/([^/]+)/suite/([^/]+)/test/([^/]+)/$' % group_and_project, views.test_run_suite_test_details, name='testrun_suite_test_details'),
     url(r'^(%s)/(%s)/build/([^/]+)/testrun/([^/]+)/suite/([^/]+)/test/([^/]+)/details/$' % group_and_project, views.test_run_suite_test_details, name='testrun_suite_test_details'),
     url(r'^(%s)/(%s)/build/([^/]+)/testrun/([^/]+)/suite/([^/]+)/metrics/$' % group_and_project, views.test_run_suite_metrics, name='testrun_suite_metrics'),
     url(r'^(%s)/(%s)/build/([^/]+)/testrun/([^/]+)/suite/([^/]+)/test/([^/]+)/log$' % group_and_project, views.test_details_log, name='test_details_log'),

squad-1.91/squad/plugins/lib/__init__.py

@@ -0,0 +1 @@
+from squad.plugins import Plugin # noqa