squad 1.65__tar.gz → 1.67__tar.gz

{squad-1.65 → squad-1.67}/CHANGELOG.md

@@ -1,3 +1,17 @@
+# 1.67
+
+This 1.77 release adds support for receiving callback events from Tuxsuite
+backends. This will allow build and test objects to be pushed directly to
+SQUAD which in turn will stop polling from Tuxsuite, saving unecessary
+requests. The release also fixes a bug when composing URLs used in github
+plugin.
+
+# 1.66
+
+This 1.66 release adds support for listening to LAVA websockets over
+ZMQ. The release also fixes infinite Tuxsuite polling when jobs run into
+infrastructure errors.
+
 # 1.65
 
 This 1.65 release adds support to cancelling tuxsuite jobs via SQUAD

{squad-1.65 → squad-1.67}/Dockerfile

@@ -6,6 +6,7 @@ RUN apt-get update -q=2 && \
     apt-get -qq install --no-install-recommends iproute2 auto-apt-proxy >/dev/null && \
     apt-get -qq install --no-install-recommends >/dev/null \
         python3 \
+        python3-aiohttp \
         python3-asgiref \
         python3-celery \
         python3-coreapi  \

{squad-1.65/squad.egg-info → squad-1.67}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: squad
-Version: 1.65
+Version: 1.67
 Summary: Software Quality Dashboard
 Home-page: https://github.com/Linaro/squad
 Author: Antonio Terceiro

{squad-1.65 → squad-1.67}/doc/ci.rst

@@ -225,8 +225,8 @@ a few limitations with this setup:
    distinguishing between identically named tests from different parts of
    multinode job.
 
-Callbacks
----------
+Callbacks Triggers
+------------------
 
 In SQUAD, callbacks can be attached to Builds. They are triggered once the given build finishes fetching all test jobs from the backend.
 
@@ -281,4 +281,49 @@ It's important to point out that:
 * The callback headers will be merged with the build's project settings if available. If header names collide, project settings will get overwritten
 * Callbacks are available in read-only mode at ``GET /api/builds/<id>/callbacks/``
 
+
+Receiving Callbacks
+-------------------
+
+SQUAD also supports receiving callbacks. Currently the use case for receiving callbacks is for
+backends that need to push results back to SQUAD while not having a live connection like LAVA
+does with ZMQ or websockets. Tuxsuite is a good example as it runs on a serverless architecture
+it doesn't provide live connections, thus needing the callback feature.
+
+The callback URL should be in format:
+
+* `POST /api/fetchjob/<group_slug>/<project_slug>/<build_version>/<environment-slug>/<backend-name>`
+
+Authetication and payloads are dependant on the backend implementation. There is currently only one
+supported backend: Tuxsuite. In the section below we will describe how this integration should work.
+
+Use case: Tuxsuite
+~~~~~~~~~~~~~~~~~~
+
+SQUAD allows callbacks to be triggered by Tuxsuite. Developers trigger builds and tests to Tuxsuite as
+they would normally do. The difference now is that they can pass a URL to be POST'ed after such build or
+test is finished. Below is an example of how to do that:
+
+.. code-block:: bash
+
+   $ tuxsuite build \
+       --git-repo https://github.com/torvalds/linux.git \
+       --git-ref master \
+       --target-arch arm \
+       --toolchain gcc-12 \
+       --kconfig tinyconfig \
+       --callback https://squad.com/api/fetchjob/tuxgroup/tuxproject/mybuild/myenv/tuxsuite
+
+This tells Tuxsuite to POST to `https://squad.com/api/fetchjob/tuxgroup/tuxproject/mybuild/myenv/tuxsuite`.
+Payload and authentication are Tuxsuite-specific and documentation can be found at https://docs.tuxsuite.com/callbacks/.
+
+In order to validate that the request is coming from Tuxsuite, SQUAD checks the `x-tux-payload-signature` header and
+match it with public key configured in each project setting page.
+
+SQUAD will attempt to read key from `TUXSUITE_PUBLIC_KEY` variable defined in the project settings of `tuxgroup/tuxproject`.
+If the request is valid, SQUAD will take in the payload provided by Tuxsuite, save it and enqueue a test job for fetching.
+
+The main difference now is that Tuxsuite will be the one telling SQUAD when to fetch results. This prevents SQUAD from polling
+Tuxsuite every now and then.
+
 .. vim: ts=4 sw=4 et=1

{squad-1.65 → squad-1.67}/requirements.txt

@@ -1,3 +1,4 @@
+aiohttp
 celery<5.0,>=4.4
 cryptography
 coreapi

{squad-1.65 → squad-1.67}/squad/api/ci.py

@@ -1,3 +1,5 @@
+import json
+
 from django.http import HttpResponse, HttpResponseBadRequest
 from django.shortcuts import get_object_or_404
 from django.views.decorators.csrf import csrf_exempt
@@ -8,6 +10,7 @@ from squad.ci.exceptions import SubmissionIssue
 from squad.ci.tasks import submit, fetch
 from squad.ci.models import Backend, TestJob
 from squad.core.utils import log_addition
+from squad.core.models import Project
 
 
 @require_http_methods(["POST"])
@@ -149,3 +152,42 @@ def resubmit_job(request, test_job_id, method='resubmit'):
 @csrf_exempt
 def force_resubmit_job(request, test_job_id):
     return resubmit_job(request, test_job_id, method='force_resubmit')
+
+
+@require_http_methods(["POST"])
+@csrf_exempt
+def fetch_job(request, group_slug, project_slug, version, environment_slug, backend_name):
+    try:
+        backend = Backend.objects.get(name=backend_name)
+    except Backend.DoesNotExist:
+        return HttpResponseBadRequest("requested backend does not exist")
+
+    if not backend.supports_callbacks():
+        return HttpResponseBadRequest("requested backend does not support callbacks")
+
+    try:
+        project = Project.objects.get(slug=project_slug, group__slug=group_slug)
+    except Project.DoesNotExist:
+        return HttpResponseBadRequest("group/project does not exist")
+
+    try:
+        backend.validate_callback(request, project)
+    except Exception as e:
+        return HttpResponseBadRequest(f"request is not valid for this backend: {e}")
+
+    environment, _ = project.environments.get_or_create(slug=environment_slug)
+    build, _ = project.builds.get_or_create(version=version)
+
+    try:
+        payload = json.loads(request.body)
+    except Exception as e:
+        return HttpResponseBadRequest(f"payload failed to parse as json: {e}")
+
+    try:
+        test_job = backend.process_callback(payload, build, environment)
+    except Exception as e:
+        return HttpResponseBadRequest(f"malformed callback payload: {e}")
+
+    fetch.delay(test_job.id)
+
+    return HttpResponse(test_job.id, status=201)

{squad-1.65 → squad-1.67}/squad/api/urls.py

@@ -21,6 +21,7 @@ urlpatterns = [
     url(r'^submit/(%s)/(%s)/(%s)/(%s)' % (group_slug_pattern, slug_pattern, slug_pattern, slug_pattern), views.add_test_run),
     url(r'^submitjob/(%s)/(%s)/(%s)/(%s)' % (group_slug_pattern, slug_pattern, slug_pattern, slug_pattern), ci.submit_job),
     url(r'^watchjob/(%s)/(%s)/(%s)/(%s)' % (group_slug_pattern, slug_pattern, slug_pattern, slug_pattern), ci.watch_job),
+    url(r'^fetchjob/(%s)/(%s)/(%s)/(%s)/(%s)' % (group_slug_pattern, slug_pattern, slug_pattern, slug_pattern, slug_pattern), ci.fetch_job),
     url(r'^data/(%s)/(%s)' % (group_slug_pattern, slug_pattern), data.get),
     url(r'^resubmit/([0-9]+)', ci.resubmit_job),
     url(r'^forceresubmit/([0-9]+)', ci.force_resubmit_job),

{squad-1.65 → squad-1.67}/squad/ci/backend/fake.py

@@ -65,3 +65,6 @@ class Backend(object):
 
     def check_job_definition(self, definition):
         return True
+
+    def supports_callbacks(self):
+        return False

{squad-1.65 → squad-1.67}/squad/ci/backend/lava.py

@@ -1,3 +1,5 @@
+import asyncio
+import aiohttp
 import json
 import re
 import requests
@@ -8,13 +10,14 @@ import yaml
 import xmlrpc
 import zmq
 
+from asgiref.sync import sync_to_async
 from dateutil.parser import isoparse
 from contextlib import contextmanager
 from io import BytesIO, TextIOWrapper, StringIO
 from zmq.utils.strtypes import u
 
 from xmlrpc import client as xmlrpclib
-from urllib.parse import urlsplit, urljoin
+from urllib.parse import urlsplit, urljoin, urlparse
 
 
 from squad.ci.models import TestJob
@@ -160,6 +163,41 @@ class Backend(BaseBackend):
                 raise FetchIssue(self.url_remove_token(str(fault)))
 
     def listen(self):
+        if not self.listen_websocket():
+            self.listen_zmq()
+
+    def listen_websocket(self):
+        async def handler():
+            url = urlparse(self.data.url)
+            ws_url = f"{url.scheme}://{self.data.username}:{self.data.token}@{url.netloc}/ws/"
+            try:
+                while True:
+                    try:
+                        async with aiohttp.ClientSession() as session:
+                            self.log_debug(f"connecting to {url.scheme}://{url.netloc}/ws/")
+                            async with session.ws_connect(ws_url, heartbeat=30) as ws:
+                                async for msg in ws:
+                                    if msg.type == aiohttp.WSMsgType.TEXT:
+                                        try:
+                                            (topic, uuid, dt, username, data) = (m for m in msg.json()[:])
+                                            data = json.loads(data)
+                                            if "error" in data:
+                                                raise aiohttp.ClientError(data["error"])
+                                        except ValueError:
+                                            continue
+                                        await sync_to_async(self.receive_event)(topic, data)
+                                await asyncio.sleep(1)
+                    except aiohttp.ClientError as e:
+                        self.log_warn(f"Failed to start client: {e}")
+                        return False
+            except Exception as e:
+                # Fall back to ZMQ
+                self.log_warn(f"Failed to maintain websocket connection: {e}")
+                return False
+
+        asyncio.run(handler())
+
+    def listen_zmq(self):
         listener_url = self.get_listener_url()
         if not listener_url:
             self.log_warn("Can't connect, no listener URL")

{squad-1.65 → squad-1.67}/squad/ci/backend/null.py

@@ -104,6 +104,25 @@ class Backend(object):
         """
         raise NotImplementedError
 
+    def supports_callbacks(self):
+        """
+        Returns True if this backend supports callbacks, False otherwise
+        """
+        return False
+
+    def validate_callback(self, request, project):
+        """
+        Raises an exception in case the request does not pass the validation
+        """
+        raise NotImplementedError
+
+    def process_callback(self, json_payload, build, environment, backend):
+        """
+        Returns a test_job if processing callback's payload fine, or raise exceptions
+        if something isn't right
+        """
+        raise NotImplementedError
+
     def format_message(self, msg):
         if self.data and hasattr(self.data, "name"):
             return self.data.name + ': ' + msg

{squad-1.65 → squad-1.67}/squad/ci/backend/tuxsuite.py

@@ -1,3 +1,4 @@
+import base64
 import hashlib
 import logging
 import re
@@ -8,8 +9,15 @@ import json
 from functools import reduce
 from urllib.parse import urljoin
 
+from cryptography.hazmat.primitives.asymmetric import ec
+from cryptography.hazmat.primitives import (
+    hashes,
+    serialization,
+)
+
 from squad.ci.backend.null import Backend as BaseBackend
 from squad.ci.exceptions import FetchIssue, TemporaryFetchIssue
+from squad.ci.models import TestJob
 
 
 logger = logging.getLogger('squad.ci.backend.tuxsuite')
@@ -83,16 +91,43 @@ class Backend(BaseBackend):
         # The regex below is supposed to find only one match
         return matches[0]
 
+    def generate_job_id(self, result_type, result):
+        """
+            The job id for TuxSuite results is generated using 3 pieces of info:
+            1. If it's either "BUILD" or "TEST" result;
+            2. The TuxSuite project. Ex: "linaro/anders"
+            3. The ksuid of the object. Ex: "1yPYGaOEPNwr2pfqBgONY43zORp"
+
+            A couple examples for job_id are:
+            - BUILD:linaro@anders#1yPYGaOEPNwr2pCqBgONY43zORq
+            - TEST:arm@bob#1yPYGaOEPNwr2pCqBgONY43zORp
+
+            Then it's up to SQUAD's TuxSuite backend to parse the job_id
+            and fetch results properly.
+        """
+        _type = "TEST" if result_type == "test" else "BUILD"
+        project = result["project"].replace("/", "@")
+        uid = result["uid"]
+        return f"{_type}:{project}#{uid}"
+
     def fetch_url(self, *urlbits):
         url = reduce(urljoin, urlbits)
 
         try:
             response = requests.get(url)
         except Exception as e:
-            raise TemporaryFetchIssue(f"Can't retrieve from {url}: %s" % e)
+            raise TemporaryFetchIssue(f"Can't retrieve from {url}: {e}")
 
         return response
 
+    def fetch_from_results_input(self, test_job):
+        try:
+            return json.loads(test_job.input)
+        except Exception as e:
+            logger.error(f"Can't parse results from job's input: {e}")
+
+        return None
+
     def parse_build_results(self, test_job, job_url, results, settings):
         required_keys = ['build_status', 'warnings_count', 'download_url', 'retry']
         self.__check_required_keys__(required_keys, results)
@@ -163,6 +198,10 @@ class Backend(BaseBackend):
         if results['result'] == 'fail':
             test_job.failure = str(results['results'])
 
+        elif results['result'] == 'error':
+            test_job.failure = 'tuxsuite infrastructure error'
+            return 'Incomplete', completed, metadata, tests, metrics, logs
+
         # If boot result is unkown, a retry is needed, otherwise, it either passed or failed
         if 'unknown' == results['results']['boot']:
             return None
@@ -175,6 +214,9 @@ class Backend(BaseBackend):
             _, _, test_id = self.parse_job_id(test_job.job_id)
             build_id = results['waiting_for']
             build_url = job_url.replace(test_id, build_id).replace('tests', 'builds')
+
+            # TODO: check if we can save a few seconds by querying a testjob that
+            # already contains build results
             build_metadata = self.fetch_url(build_url).json()
 
             build_metadata_keys = settings.get('TEST_BUILD_METADATA_KEYS', [])
@@ -207,7 +249,12 @@ class Backend(BaseBackend):
 
     def fetch(self, test_job):
         url = self.job_url(test_job)
-        results = self.fetch_url(url).json()
+        if test_job.input:
+            results = self.fetch_from_results_input(test_job)
+            test_job.input = None
+        else:
+            results = self.fetch_url(url).json()
+
         if results.get('state') != 'finished':
             return None
 
@@ -249,3 +296,49 @@ class Backend(BaseBackend):
         url = urljoin(self.data.url, endpoint)
         response = requests.post(url)
         return response.status_code == 200
+
+    def supports_callbacks(self):
+        return True
+
+    def validate_callback(self, request, project):
+        signature = request.headers.get("x-tux-payload-signature", None)
+        if signature is None:
+            raise Exception("tuxsuite request is missing signature headers")
+
+        public_key = project.get_setting("TUXSUITE_PUBLIC_KEY")
+        if public_key is None:
+            raise Exception("missing tuxsuite public key for this project")
+
+        payload = request.body
+        signature = base64.urlsafe_b64decode(signature)
+        key = serialization.load_ssh_public_key(public_key.encode("ascii"))
+        key.verify(
+            signature,
+            payload,
+            ec.ECDSA(hashes.SHA256()),
+        )
+
+    def process_callback(self, json_payload, build, environment, backend):
+        if "kind" not in json_payload or "status" not in json_payload:
+            raise Exception("`kind` and `status` are required in the payload")
+
+        kind = json_payload["kind"]
+        status = json_payload["status"]
+        job_id = self.generate_job_id(kind, status)
+        try:
+            # Tuxsuite's job id DO NOT repeat, like ever
+            testjob = TestJob.objects.get(job_id=job_id, target_build=build, environment=environment.slug)
+        except TestJob.DoesNotExist:
+            testjob = TestJob.objects.create(
+                backend=backend,
+                target=build.project,
+                target_build=build,
+                environment=environment.slug,
+                submitted=True,
+                job_id=job_id
+            )
+
+        # Saves the input so it can be processed by the queue
+        testjob.input = json.dumps(status)
+
+        return testjob

squad-1.67/squad/ci/migrations/0029_create_testjob_results_input.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2 on 2023-04-26 16:50
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ci', '0028_create_testjob_indexes'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ResultsInput',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('text', models.TextField(blank=True, null=True)),
+                ('test_job', models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='results_input', to='ci.testjob')),
+            ],
+        ),
+    ]

{squad-1.65 → squad-1.67}/squad/ci/models.py

@@ -164,6 +164,15 @@ class Backend(models.Model):
     def check_job_definition(self, definition):
         return self.get_implementation().check_job_definition(definition)
 
+    def supports_callbacks(self):
+        return self.get_implementation().supports_callbacks()
+
+    def validate_callback(self, request, project):
+        self.get_implementation().validate_callback(request, project)
+
+    def process_callback(self, payload, build, environment):
+        return self.get_implementation().process_callback(payload, build, environment, self)
+
     def __str__(self):
         return '%s (%s)' % (self.name, self.implementation_type)
 
@@ -241,6 +250,21 @@ class TestJob(models.Model):
             return self.backend.get_implementation().job_url(self)
         return None
 
+    @property
+    def input(self):
+        try:
+            return self.results_input.text
+        except ResultsInput.DoesNotExist:
+            return None
+
+    @input.setter
+    def input(self, value):
+        if value:
+            self.results_input = ResultsInput(text=value)
+            self.results_input.save()
+        else:
+            self.results_input.delete()
+
     def resubmit(self):
         ret_value = False
         if self.can_resubmit:
@@ -311,3 +335,8 @@ class TestJob(models.Model):
         indexes = [
             models.Index(fields=['submitted', 'fetched']),
         ]
+
+
+class ResultsInput(models.Model):
+    test_job = models.OneToOneField(TestJob, related_name='results_input', on_delete=models.CASCADE, null=True)
+    text = models.TextField(null=True, blank=True)

{squad-1.65 → squad-1.67}/squad/plugins/github.py

@@ -4,6 +4,7 @@ from django.conf import settings
 from squad.core.models import ProjectStatus
 from squad.core.plugins import Plugin as BasePlugin
 from squad.frontend.templatetags.squad import project_url
+from urllib.parse import urljoin
 
 
 def build_url(build):
@@ -11,7 +12,6 @@ def build_url(build):
 
 
 class Plugin(BasePlugin):
-
     @staticmethod
     def __github_post__(build, endpoint, payload):
         api_url = build.patch_source.url
@@ -22,10 +22,11 @@ class Plugin(BasePlugin):
             "Authorization": "token %s" % api_token,
         }
 
-        url = api_url + endpoint.format(
-            owner=owner,
-            repository=repository,
-            commit=commit
+        url = urljoin(
+            api_url, endpoint.format(
+                owner=owner,
+                repository=repository,
+                commit=commit)
         )
         return requests.post(url, headers=headers, json=payload)
 

squad-1.67/squad/version.py

@@ -0,0 +1 @@
+__version__ = '1.67'

{squad-1.65 → squad-1.67/squad.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: squad
-Version: 1.65
+Version: 1.67
 Summary: Software Quality Dashboard
 Home-page: https://github.com/Linaro/squad
 Author: Antonio Terceiro

{squad-1.65 → squad-1.67}/squad.egg-info/SOURCES.txt

@@ -145,6 +145,7 @@ squad/ci/migrations/0025_backend_listen_enabled.py
 squad/ci/migrations/0026_job_start_end_time.py
 squad/ci/migrations/0027_add_tuxsuite_implementation_type.py
 squad/ci/migrations/0028_create_testjob_indexes.py
+squad/ci/migrations/0029_create_testjob_results_input.py
 squad/ci/migrations/__init__.py
 squad/ci/templates/squad/ci/testjob_resubmit.html.jinja2
 squad/ci/templates/squad/ci/testjob_resubmit.txt.jinja2

{squad-1.65 → squad-1.67}/squad.egg-info/requires.txt

@@ -1,3 +1,4 @@
+aiohttp
 celery<5.0,>=4.4
 cryptography
 coreapi

{squad-1.65 → squad-1.67}/test/api/test_ci.py

@@ -41,6 +41,7 @@ class CiApiTest(TestCase):
         Token.objects.create(user=self.project_admin_user, key='adminkey')
 
         self.backend = models.Backend.objects.create(name='lava', implementation_type='fake')
+        self.tuxsuite = models.Backend.objects.create(name='tuxsuite', implementation_type='tuxsuite')
         self.client = APIClient('thekey')
         self.submitter_client = APIClient('thesubmitterkey')
         self.restclient = RestAPIClient('thekey')
@@ -302,6 +303,58 @@ class CiApiTest(TestCase):
         r = self.client.post('/api/watchjob/mygroup/myproject/1/myenv', args)
         self.assertEqual(400, r.status_code)
 
+    def test_fetch_testjob_not_supported_by_backend(self):
+        # Backend does not exist
+        r = self.client.post('/api/fetchjob/mygroup/myproject/1/myenv/does-not-exist')
+        self.assertEqual(400, r.status_code)
+
+        # Backend does not support callbacks
+        r = self.client.post('/api/fetchjob/mygroup/myproject/1/myenv/lava')
+        self.assertEqual(400, r.status_code)
+
+    def test_fetch_testjob_group_project_does_not_exist(self):
+        r = self.client.post('/api/fetchjob/doesnotexist/myproject/1/myenv/lava')
+        self.assertEqual(400, r.status_code)
+
+        r = self.client.post('/api/fetchjob/mygroup/doesnotexist/1/myenv/lava')
+        self.assertEqual(400, r.status_code)
+
+    @patch("squad.ci.tasks.fetch.delay")
+    @patch("squad.ci.backend.tuxsuite.Backend.process_callback")
+    @patch("squad.ci.backend.tuxsuite.Backend.validate_callback")
+    def test_fetch_testjob_invalid(self, mock_validate_callback, mock_process_callback, mock_fetch):
+        mock_validate_callback.side_effect = Exception("bad")
+        r = self.client.post('/api/fetchjob/mygroup/myproject/1/myenv/tuxsuite')
+        self.assertEqual(400, r.status_code)
+        mock_validate_callback.assert_called()
+        mock_validate_callback.reset_mock()
+
+        mock_validate_callback.side_effect = None
+        r = self.client.post('/api/fetchjob/mygroup/myproject/1/myenv/tuxsuite')
+        self.assertEqual(400, r.status_code)
+        mock_validate_callback.assert_called()
+        mock_validate_callback.reset_mock()
+
+        mock_process_callback.side_effect = Exception("bad")
+        r = self.client.post('/api/fetchjob/mygroup/myproject/1/myenv/tuxsuite', {"a": 1}, content_type="application/json")
+        self.assertEqual(400, r.status_code)
+        mock_validate_callback.assert_called()
+        mock_process_callback.assert_called()
+
+        mock_fetch.assert_not_called()
+
+    @patch("squad.ci.tasks.fetch.delay")
+    @patch("squad.ci.backend.tuxsuite.Backend.process_callback")
+    @patch("squad.ci.backend.tuxsuite.Backend.validate_callback")
+    def test_fetch_testjob(self, mock_validate_callback, mock_process_callback, mock_fetch):
+        testjob = models.TestJob(id=123)
+        mock_process_callback.return_value = testjob
+        r = self.client.post('/api/fetchjob/mygroup/myproject/1/myenv/tuxsuite', {"a": 1}, content_type="application/json")
+        self.assertEqual(201, r.status_code)
+        mock_validate_callback.assert_called()
+        mock_process_callback.assert_called()
+        mock_fetch.assert_called_with(testjob.id)
+
     @patch('squad.ci.models.Backend.get_implementation')
     def test_resubmit_submitter(self, get_implementation):
         impl = MagicMock()