squad 1.66__py3-none-any.whl → 1.67__py3-none-any.whl

squad/api/ci.py

@@ -1,3 +1,5 @@
+import json
+
 from django.http import HttpResponse, HttpResponseBadRequest
 from django.shortcuts import get_object_or_404
 from django.views.decorators.csrf import csrf_exempt
@@ -8,6 +10,7 @@ from squad.ci.exceptions import SubmissionIssue
 from squad.ci.tasks import submit, fetch
 from squad.ci.models import Backend, TestJob
 from squad.core.utils import log_addition
+from squad.core.models import Project
 
 
 @require_http_methods(["POST"])
@@ -149,3 +152,42 @@ def resubmit_job(request, test_job_id, method='resubmit'):
 @csrf_exempt
 def force_resubmit_job(request, test_job_id):
     return resubmit_job(request, test_job_id, method='force_resubmit')
+
+
+@require_http_methods(["POST"])
+@csrf_exempt
+def fetch_job(request, group_slug, project_slug, version, environment_slug, backend_name):
+    try:
+        backend = Backend.objects.get(name=backend_name)
+    except Backend.DoesNotExist:
+        return HttpResponseBadRequest("requested backend does not exist")
+
+    if not backend.supports_callbacks():
+        return HttpResponseBadRequest("requested backend does not support callbacks")
+
+    try:
+        project = Project.objects.get(slug=project_slug, group__slug=group_slug)
+    except Project.DoesNotExist:
+        return HttpResponseBadRequest("group/project does not exist")
+
+    try:
+        backend.validate_callback(request, project)
+    except Exception as e:
+        return HttpResponseBadRequest(f"request is not valid for this backend: {e}")
+
+    environment, _ = project.environments.get_or_create(slug=environment_slug)
+    build, _ = project.builds.get_or_create(version=version)
+
+    try:
+        payload = json.loads(request.body)
+    except Exception as e:
+        return HttpResponseBadRequest(f"payload failed to parse as json: {e}")
+
+    try:
+        test_job = backend.process_callback(payload, build, environment)
+    except Exception as e:
+        return HttpResponseBadRequest(f"malformed callback payload: {e}")
+
+    fetch.delay(test_job.id)
+
+    return HttpResponse(test_job.id, status=201)

squad/api/urls.py

@@ -21,6 +21,7 @@ urlpatterns = [
     url(r'^submit/(%s)/(%s)/(%s)/(%s)' % (group_slug_pattern, slug_pattern, slug_pattern, slug_pattern), views.add_test_run),
     url(r'^submitjob/(%s)/(%s)/(%s)/(%s)' % (group_slug_pattern, slug_pattern, slug_pattern, slug_pattern), ci.submit_job),
     url(r'^watchjob/(%s)/(%s)/(%s)/(%s)' % (group_slug_pattern, slug_pattern, slug_pattern, slug_pattern), ci.watch_job),
+    url(r'^fetchjob/(%s)/(%s)/(%s)/(%s)/(%s)' % (group_slug_pattern, slug_pattern, slug_pattern, slug_pattern, slug_pattern), ci.fetch_job),
     url(r'^data/(%s)/(%s)' % (group_slug_pattern, slug_pattern), data.get),
     url(r'^resubmit/([0-9]+)', ci.resubmit_job),
     url(r'^forceresubmit/([0-9]+)', ci.force_resubmit_job),

squad/ci/backend/fake.py

@@ -65,3 +65,6 @@ class Backend(object):
 
     def check_job_definition(self, definition):
         return True
+
+    def supports_callbacks(self):
+        return False

squad/ci/backend/null.py

@@ -104,6 +104,25 @@ class Backend(object):
         """
         raise NotImplementedError
 
+    def supports_callbacks(self):
+        """
+        Returns True if this backend supports callbacks, False otherwise
+        """
+        return False
+
+    def validate_callback(self, request, project):
+        """
+        Raises an exception in case the request does not pass the validation
+        """
+        raise NotImplementedError
+
+    def process_callback(self, json_payload, build, environment, backend):
+        """
+        Returns a test_job if processing callback's payload fine, or raise exceptions
+        if something isn't right
+        """
+        raise NotImplementedError
+
     def format_message(self, msg):
         if self.data and hasattr(self.data, "name"):
             return self.data.name + ': ' + msg

squad/ci/backend/tuxsuite.py

@@ -1,3 +1,4 @@
+import base64
 import hashlib
 import logging
 import re
@@ -8,8 +9,15 @@ import json
 from functools import reduce
 from urllib.parse import urljoin
 
+from cryptography.hazmat.primitives.asymmetric import ec
+from cryptography.hazmat.primitives import (
+    hashes,
+    serialization,
+)
+
 from squad.ci.backend.null import Backend as BaseBackend
 from squad.ci.exceptions import FetchIssue, TemporaryFetchIssue
+from squad.ci.models import TestJob
 
 
 logger = logging.getLogger('squad.ci.backend.tuxsuite')
@@ -83,16 +91,43 @@ class Backend(BaseBackend):
         # The regex below is supposed to find only one match
         return matches[0]
 
+    def generate_job_id(self, result_type, result):
+        """
+            The job id for TuxSuite results is generated using 3 pieces of info:
+            1. If it's either "BUILD" or "TEST" result;
+            2. The TuxSuite project. Ex: "linaro/anders"
+            3. The ksuid of the object. Ex: "1yPYGaOEPNwr2pfqBgONY43zORp"
+
+            A couple examples for job_id are:
+            - BUILD:linaro@anders#1yPYGaOEPNwr2pCqBgONY43zORq
+            - TEST:arm@bob#1yPYGaOEPNwr2pCqBgONY43zORp
+
+            Then it's up to SQUAD's TuxSuite backend to parse the job_id
+            and fetch results properly.
+        """
+        _type = "TEST" if result_type == "test" else "BUILD"
+        project = result["project"].replace("/", "@")
+        uid = result["uid"]
+        return f"{_type}:{project}#{uid}"
+
     def fetch_url(self, *urlbits):
         url = reduce(urljoin, urlbits)
 
         try:
             response = requests.get(url)
         except Exception as e:
-            raise TemporaryFetchIssue(f"Can't retrieve from {url}: %s" % e)
+            raise TemporaryFetchIssue(f"Can't retrieve from {url}: {e}")
 
         return response
 
+    def fetch_from_results_input(self, test_job):
+        try:
+            return json.loads(test_job.input)
+        except Exception as e:
+            logger.error(f"Can't parse results from job's input: {e}")
+
+        return None
+
     def parse_build_results(self, test_job, job_url, results, settings):
         required_keys = ['build_status', 'warnings_count', 'download_url', 'retry']
         self.__check_required_keys__(required_keys, results)
@@ -179,6 +214,9 @@ class Backend(BaseBackend):
             _, _, test_id = self.parse_job_id(test_job.job_id)
             build_id = results['waiting_for']
             build_url = job_url.replace(test_id, build_id).replace('tests', 'builds')
+
+            # TODO: check if we can save a few seconds by querying a testjob that
+            # already contains build results
             build_metadata = self.fetch_url(build_url).json()
 
             build_metadata_keys = settings.get('TEST_BUILD_METADATA_KEYS', [])
@@ -211,7 +249,12 @@ class Backend(BaseBackend):
 
     def fetch(self, test_job):
         url = self.job_url(test_job)
-        results = self.fetch_url(url).json()
+        if test_job.input:
+            results = self.fetch_from_results_input(test_job)
+            test_job.input = None
+        else:
+            results = self.fetch_url(url).json()
+
         if results.get('state') != 'finished':
             return None
 
@@ -253,3 +296,49 @@ class Backend(BaseBackend):
         url = urljoin(self.data.url, endpoint)
         response = requests.post(url)
         return response.status_code == 200
+
+    def supports_callbacks(self):
+        return True
+
+    def validate_callback(self, request, project):
+        signature = request.headers.get("x-tux-payload-signature", None)
+        if signature is None:
+            raise Exception("tuxsuite request is missing signature headers")
+
+        public_key = project.get_setting("TUXSUITE_PUBLIC_KEY")
+        if public_key is None:
+            raise Exception("missing tuxsuite public key for this project")
+
+        payload = request.body
+        signature = base64.urlsafe_b64decode(signature)
+        key = serialization.load_ssh_public_key(public_key.encode("ascii"))
+        key.verify(
+            signature,
+            payload,
+            ec.ECDSA(hashes.SHA256()),
+        )
+
+    def process_callback(self, json_payload, build, environment, backend):
+        if "kind" not in json_payload or "status" not in json_payload:
+            raise Exception("`kind` and `status` are required in the payload")
+
+        kind = json_payload["kind"]
+        status = json_payload["status"]
+        job_id = self.generate_job_id(kind, status)
+        try:
+            # Tuxsuite's job id DO NOT repeat, like ever
+            testjob = TestJob.objects.get(job_id=job_id, target_build=build, environment=environment.slug)
+        except TestJob.DoesNotExist:
+            testjob = TestJob.objects.create(
+                backend=backend,
+                target=build.project,
+                target_build=build,
+                environment=environment.slug,
+                submitted=True,
+                job_id=job_id
+            )
+
+        # Saves the input so it can be processed by the queue
+        testjob.input = json.dumps(status)
+
+        return testjob

squad/ci/migrations/0029_create_testjob_results_input.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2 on 2023-04-26 16:50
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ci', '0028_create_testjob_indexes'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ResultsInput',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('text', models.TextField(blank=True, null=True)),
+                ('test_job', models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='results_input', to='ci.testjob')),
+            ],
+        ),
+    ]

squad/ci/models.py

@@ -164,6 +164,15 @@ class Backend(models.Model):
     def check_job_definition(self, definition):
         return self.get_implementation().check_job_definition(definition)
 
+    def supports_callbacks(self):
+        return self.get_implementation().supports_callbacks()
+
+    def validate_callback(self, request, project):
+        self.get_implementation().validate_callback(request, project)
+
+    def process_callback(self, payload, build, environment):
+        return self.get_implementation().process_callback(payload, build, environment, self)
+
     def __str__(self):
         return '%s (%s)' % (self.name, self.implementation_type)
 
@@ -241,6 +250,21 @@ class TestJob(models.Model):
             return self.backend.get_implementation().job_url(self)
         return None
 
+    @property
+    def input(self):
+        try:
+            return self.results_input.text
+        except ResultsInput.DoesNotExist:
+            return None
+
+    @input.setter
+    def input(self, value):
+        if value:
+            self.results_input = ResultsInput(text=value)
+            self.results_input.save()
+        else:
+            self.results_input.delete()
+
     def resubmit(self):
         ret_value = False
         if self.can_resubmit:
@@ -311,3 +335,8 @@ class TestJob(models.Model):
         indexes = [
             models.Index(fields=['submitted', 'fetched']),
         ]
+
+
+class ResultsInput(models.Model):
+    test_job = models.OneToOneField(TestJob, related_name='results_input', on_delete=models.CASCADE, null=True)
+    text = models.TextField(null=True, blank=True)

squad/plugins/github.py

@@ -4,6 +4,7 @@ from django.conf import settings
 from squad.core.models import ProjectStatus
 from squad.core.plugins import Plugin as BasePlugin
 from squad.frontend.templatetags.squad import project_url
+from urllib.parse import urljoin
 
 
 def build_url(build):
@@ -11,7 +12,6 @@ def build_url(build):
 
 
 class Plugin(BasePlugin):
-
     @staticmethod
     def __github_post__(build, endpoint, payload):
         api_url = build.patch_source.url
@@ -22,10 +22,11 @@ class Plugin(BasePlugin):
             "Authorization": "token %s" % api_token,
         }
 
-        url = api_url + endpoint.format(
-            owner=owner,
-            repository=repository,
-            commit=commit
+        url = urljoin(
+            api_url, endpoint.format(
+                owner=owner,
+                repository=repository,
+                commit=commit)
         )
         return requests.post(url, headers=headers, json=payload)
 

squad/version.py

@@ -1 +1 @@
-__version__ = '1.66'
+__version__ = '1.67'

{squad-1.66.dist-info → squad-1.67.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: squad
-Version: 1.66
+Version: 1.67
 Summary: Software Quality Dashboard
 Home-page: https://github.com/Linaro/squad
 Author: Antonio Terceiro

{squad-1.66.dist-info → squad-1.67.dist-info}/RECORD

@@ -10,29 +10,29 @@ squad/manage.py,sha256=Z-LXT67p0R-IzwJ9fLIAacEZmU0VUjqDOSg7j2ZSxJ4,1437
 squad/settings.py,sha256=EzUgd8Egzp6GrZd-Vx6OlAorhlVavT84bxvsqD58oZg,14051
 squad/socialaccount.py,sha256=vySqPwQ3qVVpahuJ-Snln8K--yzRL3bw4Nx27AsB39A,789
 squad/urls.py,sha256=JiEfVW8YlzLPE52c2aHzdn5kVVKK4o22w8h5KOA6QhQ,2776
-squad/version.py,sha256=UZgUaZ1h7d5_HEpkVjhFfumarSXOI5FTWWBKqnCojyU,21
+squad/version.py,sha256=cyth8cQRyIgeHOgLyFVNnADaoVSn0Rrc5rl9Qs7F8gs,21
 squad/wsgi.py,sha256=SF8T0cQ0OPVyuYjO5YXBIQzvSXQHV0M2BTmd4gP1rPs,387
 squad/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 squad/api/apps.py,sha256=Trk72p-iV1uGn0o5mdJn5HARUoHGbfgO49jwXvpkmdQ,141
-squad/api/ci.py,sha256=lglQch5iOeGRu6zdjLss2nkPVxer_vDFqBOIlhFMWPs,5114
+squad/api/ci.py,sha256=7eJvUwUbQ7sJzUxcxYURDV1tOT1ouqo0M3L7ojWVAFE,6536
 squad/api/data.py,sha256=obKDV0-neEvj5lPF9VED2gy_hpfhGtLJABYvSY38ing,2379
 squad/api/filters.py,sha256=Zvp8DCJmiNquFWqvfVseEAAMYYPiT95RUjqKdzcqSnw,6917
 squad/api/rest.py,sha256=aVHqq-mXLctPjro29miLqWmR1gMEAIMBeXdbZX-7rQI,74991
-squad/api/urls.py,sha256=bud5wHWbHes-klZ7rzBuF7wkKsctDzPxI406I-EwEHI,1202
+squad/api/urls.py,sha256=rmsdaL1uOCVSZ5x1redup9RliICmijaBjRK5ObsTkG8,1343
 squad/api/utils.py,sha256=Sa8QFId3_oSqD2UOoY3Kuh54LLDLPNMq2sub5ktd6Fs,1160
 squad/api/views.py,sha256=yGLUp6RtNI5vuae6cOStMuUpSia46LcEVam3eMXmEqY,3885
 squad/ci/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 squad/ci/admin.py,sha256=7yB-6F0cvt0NVvzGOTlZCyGPV_YHarmbKJZTTzataT4,2255
 squad/ci/apps.py,sha256=6OVnzTdJkxdqEJnKWYE9dZgUcc29_T1LrDw41cK4EQk,139
 squad/ci/exceptions.py,sha256=a1sccygniTYDSQi7FRn_6doapddFFiMf55AwGUh5Y80,227
-squad/ci/models.py,sha256=1Cwe7RBtftXl7T98GBHzmps8rgqE2prDZJ1RtP4k4zk,11379
+squad/ci/models.py,sha256=RoXJY4Xgu3-YA4FtcK-kem8HR8paqOb5648NByAlzDo,12325
 squad/ci/tasks.py,sha256=yrtxfPuYEqqGCDYRwLz2XyIp9a7LJ-K6Zm3VS1ymdZk,2728
 squad/ci/utils.py,sha256=38zHpw8xkZDSFlkG-2BwSK6AkcddK9OkN9LXuQ3SHR0,97
 squad/ci/backend/__init__.py,sha256=yhpotXT9F4IdAOXvGQ3-17eOHAFwoaqf9SnMX17ab30,534
-squad/ci/backend/fake.py,sha256=jVFwriX947DJKplwYD2cKW9pT3p1C7KGwPYROS7wxf0,2103
+squad/ci/backend/fake.py,sha256=zzOXGesDCW9xiMQvXGD_jqCQF32yEd7hPM8DgfZxUok,2159
 squad/ci/backend/lava.py,sha256=InqmLjf_txjKpMDpmsYbkPJEhOVADBGnTzwaA76fr1Y,33076
-squad/ci/backend/null.py,sha256=AzACuhYP0bsZi_OzDEJs_JvkqjczpVjnrgcQLV-OEgg,4356
-squad/ci/backend/tuxsuite.py,sha256=Oud5rjvZ99ImNzwne3GfgPbeFGX-USyidRLmcNoqAJI,9555
+squad/ci/backend/null.py,sha256=vP77Xj7roruehSjX8fJs7xK2aWxgaUA2id3P8nHNrEY,4949
+squad/ci/backend/tuxsuite.py,sha256=m4Hgpb0twM4icOrPdEHUgvezmVq9DDvtEloojit_2pg,12759
 squad/ci/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 squad/ci/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 squad/ci/management/commands/create_tuxsuite_boot_tests.py,sha256=JvjNusebLX71eyz9d-kaeCyekYSpzc1eXoeIqWK9ygo,4045
@@ -66,6 +66,7 @@ squad/ci/migrations/0025_backend_listen_enabled.py,sha256=t7Tx7URhsz-Q4GGuoKYNJY
 squad/ci/migrations/0026_job_start_end_time.py,sha256=18swRRnDXIhHpr2ykMFns04VS1_Fbs7Zdc0HOrikSSY,543
 squad/ci/migrations/0027_add_tuxsuite_implementation_type.py,sha256=5Max0wE_jnU5FXdJzbrPGr73N0pdZlA321pZKE-yDtE,502
 squad/ci/migrations/0028_create_testjob_indexes.py,sha256=VYT2wmrvjHJLOazBXZb15vKzGJn8jgKCVNNSSj4Nct4,612
+squad/ci/migrations/0029_create_testjob_results_input.py,sha256=Ax0jBSthCEDWUp7tLfdCgwa0nXgC2JDXOdQ26Kl9_fA,712
 squad/ci/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 squad/ci/templates/squad/ci/testjob_resubmit.html.jinja2,sha256=ys8zb4E2CJSsi4-uP57glkOFhWVK5x8-yUZcDdi81sY,1693
 squad/ci/templates/squad/ci/testjob_resubmit.txt.jinja2,sha256=yKYtixlO5zWh6ChD_mk8yDQly_iQae5slB7a0FwNPv0,610
@@ -422,16 +423,16 @@ squad/frontend/templatetags/squad.py,sha256=YSDgZu8e6e99rFhHudzVKh-6cIfoeq5fC0lW
 squad/plugins/__init__.py,sha256=9BSzy2jFIoDpWlhD7odPPrLdW4CC3btBhdFCvB651dM,152
 squad/plugins/example.py,sha256=BKpwd315lHRIuNXJPteibpwfnI6C5eXYHYdFYBtVmsI,89
 squad/plugins/gerrit.py,sha256=CqO2KnFQzu9utr_TQ-sGr1wg3ln0B-bS2-c0_i8T5-c,7009
-squad/plugins/github.py,sha256=e-TbWYcPK1r3HnYj3Y-eQ4zdBW4fRO2eBvWvE4CjxeI,2348
+squad/plugins/github.py,sha256=4ZXhR-eBVdmLUK-dBJVMRHOyjue9oknNrUfkuUqkhY0,2413
 squad/plugins/linux_log_parser.py,sha256=Dfcvh1-t5380QS_tvNIpzW_UD-4gptT9I4FUHVbb0PU,5503
 squad/run/__init__.py,sha256=ssE8GPAGFiK6V0WpZYowav6Zqsd63dfDMMYasNa1sQg,1410
 squad/run/__main__.py,sha256=DOl8JOi4Yg7DdtwnUeGqtYBJ6P2k-D2psAEuYOjWr8w,66
 squad/run/listener.py,sha256=jBeOQhPGb4EdIREB1QsCzYuumsfJ-TqJPd3nR-0m59g,200
 squad/run/scheduler.py,sha256=CDJG3q5C0GuQuxwlMOfWTSSJpDdwbR6rzpbJfuA0xuw,277
 squad/run/worker.py,sha256=jtML0h5qKDuSbpJ6_rpWP4MT_rsGA7a24AhwGxBquzk,594
-squad-1.66.dist-info/COPYING,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
-squad-1.66.dist-info/METADATA,sha256=5oU0z0GN7cXjL6N6b5yV7ipON8CHroonQehF5rMf5Vo,1267
-squad-1.66.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-squad-1.66.dist-info/entry_points.txt,sha256=apCDQydHZtvqV334ql6NhTJUAJeZRdtAm0TVcbbAi5Q,194
-squad-1.66.dist-info/top_level.txt,sha256=_x9uqE1XppiiytmVTl_qNgpnXus6Gsef69HqfliE7WI,6
-squad-1.66.dist-info/RECORD,,
+squad-1.67.dist-info/COPYING,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
+squad-1.67.dist-info/METADATA,sha256=PA2cyjXXMPX2KVIyIzo6xcLtGugZFDHfLmqvpN8TEvU,1267
+squad-1.67.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+squad-1.67.dist-info/entry_points.txt,sha256=apCDQydHZtvqV334ql6NhTJUAJeZRdtAm0TVcbbAi5Q,194
+squad-1.67.dist-info/top_level.txt,sha256=_x9uqE1XppiiytmVTl_qNgpnXus6Gsef69HqfliE7WI,6
+squad-1.67.dist-info/RECORD,,