squad 1.79__tar.gz → 1.80__tar.gz

{squad-1.79 → squad-1.80}/CHANGELOG.md

@@ -1,3 +1,12 @@
+# 1.80
+
+This 1.80 release adds Tuxsuite new metadata when fetching tests.
+
+The release also changes a bit how SQUAD does postprocessing of testjobs.
+There was a bug in SQUAD that would cause inconsistencies if SQUAD was deployed
+in a auto-scalable environment. The bug is resolved by creating a separate
+queue called "ci_fetch_postprocess" to be consumed by a persistent worker.
+
 # 1.79
 
 This 1.79 release expands the Tuxsuite backend regex to accept groups and projects

{squad-1.79 → squad-1.80}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: squad
-Version: 1.79
+Version: 1.80
 Summary: Software Quality Dashboard
 Home-page: https://github.com/Linaro/squad
 Author: Antonio Terceiro

{squad-1.79 → squad-1.80}/squad/ci/backend/tuxsuite.py

@@ -287,6 +287,17 @@ class Backend(BaseBackend):
 
         return status, completed, metadata, tests, metrics, logs
 
+    def update_metadata_from_file(self, results, metadata):
+        if "download_url" in results:
+            download_url = results["download_url"]
+            try:
+                test_metadata_response = self.fetch_url(download_url + '/' + 'metadata.json')
+                # If fetching the metadata file did not error, decode it as json
+                if test_metadata_response.ok:
+                    metadata.update(test_metadata_response.json())
+            except TemporaryFetchIssue:
+                pass
+
     def parse_test_results(self, test_job, job_url, results, settings):
         status = 'Complete'
         completed = True
@@ -297,6 +308,10 @@ class Backend(BaseBackend):
         # Pick up some metadata from results
         metadata_keys = settings.get('TEST_METADATA_KEYS', [])
         metadata = {k: results.get(k) for k in metadata_keys}
+
+        # Add extra metadata from metadata file if it exists
+        self.update_metadata_from_file(results=results, metadata=metadata)
+
         metadata['job_url'] = job_url
         metadata['job_id'] = test_job.job_id
 

{squad-1.79 → squad-1.80}/squad/ci/models.py

@@ -143,17 +143,18 @@ class Backend(models.Model):
         except DuplicatedTestJob as exception:
             logger.error('Failed to fetch test_job(%d): "%s"' % (test_job.id, str(exception)))
 
-        if test_job.testrun:
-            self.__postprocess_testjob__(test_job)
-
-        # Remove the 'Fetching' job_status only after eventual plugins
-        # are finished, this garantees extra tests and metadata to
-        # be in SQUAD before the build is considered finished
-        test_job.job_status = status
-        test_job.save()
+        if test_job.testrun and test_job.target.enabled_plugins and any(test_job.target.enabled_plugins):
+            # Avoids cyclic import errors
+            from squad.ci.tasks import postprocess_testjob
+            test_job.save()
+            postprocess_testjob.delay(test_job.id, status)
+        else:
+            # Remove the 'Fetching' job_status only after all work is done
+            test_job.job_status = status
+            test_job.save()
 
-        if test_job.testrun:
-            UpdateProjectStatus()(test_job.testrun)
+            if test_job.testrun:
+                UpdateProjectStatus()(test_job.testrun)
 
     def __postprocess_testjob__(self, test_job):
         project = test_job.target

{squad-1.79 → squad-1.80}/squad/ci/tasks.py

@@ -1,4 +1,5 @@
 from squad.celery import app as celery
+from squad.core.tasks import UpdateProjectStatus
 from squad.ci.models import Backend, TestJob
 from squad.ci.exceptions import SubmissionIssue
 from squad.ci.utils import task_id
@@ -89,3 +90,24 @@ def send_testjob_resubmit_admin_email(job_id, resubmitted_job_id):
     if test_job.target.html_mail:
         message.attach_alternative(html_message, "text/html")
     message.send()
+
+
+@celery.task
+def postprocess_testjob(job_id, status):
+    logger.info("postprocessing  %s" % job_id)
+    testjob = TestJob.objects.get(pk=job_id)
+    backend = testjob.backend
+    backend.__postprocess_testjob__(testjob)
+
+    # Some plugins, like tradefed, can still trigger sub-tasks creating tests,
+    # ending the current thread, thus triggering build-finished events
+    # we need a way to tell the plugins to wait for all sub-tasks to be finished
+
+    # Remove the 'Fetching' job_status only after eventual plugins
+    # are finished, this garantees extra tests and metadata to
+    # be in SQUAD before the build is considered finished
+    testjob.job_status = status
+    testjob.save()
+
+    if testjob.testrun:
+        UpdateProjectStatus()(testjob.testrun)

{squad-1.79 → squad-1.80}/squad/settings.py

@@ -384,6 +384,7 @@ CELERY_TASK_ROUTES = {
     'squad.core.tasks.notification.*': {'queue': 'core_notification'},
     'squad.ci.tasks.poll': {'queue': 'ci_poll'},
     'squad.ci.tasks.fetch': {'queue': 'ci_fetch'},
+    'squad.ci.tasks.postprocess_testjob': {'queue': 'ci_fetch_postprocess'},
     'squad.ci.tasks.submit': {'queue': 'ci_quick'},
     'squad.ci.tasks.send_testjob_resubmit_admin_email': {'queue': 'ci_quick'},
 }

squad-1.80/squad/version.py

@@ -0,0 +1 @@
+__version__ = '1.80'

{squad-1.79 → squad-1.80}/squad.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: squad
-Version: 1.79
+Version: 1.80
 Summary: Software Quality Dashboard
 Home-page: https://github.com/Linaro/squad
 Author: Antonio Terceiro

{squad-1.79 → squad-1.80}/test/ci/backend/test_tuxsuite.py

@@ -138,6 +138,85 @@ class TuxSuiteTest(TestCase):
         expected = urljoin(TUXSUITE_URL, endpoint)
         self.assertEqual(expected, self.tuxsuite.job_url(testjob))
 
+    def test_parse_test_no_metadata(self):
+        results = {
+            'download_url': 'http://builds.tuxbuild.com/123',
+        }
+
+        metadata = dict()
+        expected = dict()
+        with requests_mock.Mocker() as fake_request:
+            fake_request.get(results['download_url'] + '/' + 'metadata.json', status_code=404)
+            self.tuxsuite.update_metadata_from_file(results=results, metadata=metadata)
+
+        self.assertEqual(expected, metadata)
+
+    def test_parse_test_metadata(self):
+        results = {
+            'download_url': 'http://builds.tuxbuild.com/123',
+        }
+
+        metadata = {
+            "example_metadata": "blah",
+        }
+
+        metadata_file = {
+            "arch": "arm64",
+            "host_arch": "amd64",
+            "qemu_version": "1:8.1.2+ds-1",
+            "artefacts": {
+                "rootfs": {
+                    "url": "https://storage.tuxboot.com/debian/bookworm/arm64/rootfs.ext4.xz",
+                    "sha256sum": "5e0a9ec562ffea3d9705834677df1cd43ff1ba44228b46734e10a5e990c2c169",
+                },
+                "kernel": {
+                    "url": "https://storage.tuxsuite.com/public/linaro/lkft/builds/2ZPQut79EaVwA8ANRJp7xaAVkpP/Image.gz",
+                    "sha256sum": "ab5d2ef97d7a7da95032899c3ee9233dcdf75a4091ec03d7ae2d53f05d24e114",
+                },
+                "modules": {
+                    "url": "https://storage.tuxsuite.com/public/linaro/lkft/builds/2ZPQut79EaVwA8ANRJp7xaAVkpP/modules.tar.xz",
+                    "sha256sum": "1602a287bb54f43e9d4e589c9a773cbd7b9d1bee336501792092a25d76d0f3fc",
+                },
+                "overlay-00": {
+                    "url": "https://storage.tuxboot.com/overlays/debian/bookworm/arm64/ltp/20230929/ltp.tar.xz",
+                    "sha256sum": "94ff90b59487ceb765b09a53d6642ce0e39deaa92062687355a99de3652130e0",
+                },
+            },
+            "durations": {"tests": {"ltp-controllers": "4250.66", "boot": "62.69"}},
+        }
+
+        expected = {
+            "example_metadata": "blah",
+            "arch": "arm64",
+            "host_arch": "amd64",
+            "qemu_version": "1:8.1.2+ds-1",
+            "artefacts": {
+                "rootfs": {
+                    "url": "https://storage.tuxboot.com/debian/bookworm/arm64/rootfs.ext4.xz",
+                    "sha256sum": "5e0a9ec562ffea3d9705834677df1cd43ff1ba44228b46734e10a5e990c2c169",
+                },
+                "kernel": {
+                    "url": "https://storage.tuxsuite.com/public/linaro/lkft/builds/2ZPQut79EaVwA8ANRJp7xaAVkpP/Image.gz",
+                    "sha256sum": "ab5d2ef97d7a7da95032899c3ee9233dcdf75a4091ec03d7ae2d53f05d24e114",
+                },
+                "modules": {
+                    "url": "https://storage.tuxsuite.com/public/linaro/lkft/builds/2ZPQut79EaVwA8ANRJp7xaAVkpP/modules.tar.xz",
+                    "sha256sum": "1602a287bb54f43e9d4e589c9a773cbd7b9d1bee336501792092a25d76d0f3fc",
+                },
+                "overlay-00": {
+                    "url": "https://storage.tuxboot.com/overlays/debian/bookworm/arm64/ltp/20230929/ltp.tar.xz",
+                    "sha256sum": "94ff90b59487ceb765b09a53d6642ce0e39deaa92062687355a99de3652130e0",
+                },
+            },
+            "durations": {"tests": {"ltp-controllers": "4250.66", "boot": "62.69"}},
+        }
+
+        with requests_mock.Mocker() as fake_request:
+            fake_request.get(results["download_url"] + '/' + 'metadata.json', json=metadata_file)
+            self.tuxsuite.update_metadata_from_file(results=results, metadata=metadata)
+
+        self.assertEqual(expected, metadata)
+
     def test_fetch_url(self):
         expected_logs = 'dummy build log'
 
@@ -602,6 +681,10 @@ class TuxSuiteTest(TestCase):
             'ltp-smoke/ping602': 'skip'
         }
 
+        job_data = {
+            'download_url': 'http://storage.tuxapi.com/mystorage'
+        }
+
         expected_metrics = {}
 
         testjob.input = json.dumps(test_results)
@@ -609,6 +692,7 @@ class TuxSuiteTest(TestCase):
             fake_request.get(build_url, json=build_results)
             fake_request.get(urljoin(test_url + '/', 'logs'), text=test_logs)
             fake_request.get(urljoin(test_url + '/', 'results'), json=test_results_json)
+            fake_request.get(test_url, json=job_data)
 
             status, completed, metadata, tests, metrics, logs = self.tuxsuite.fetch(testjob)
             self.assertEqual('Complete', status)

{squad-1.79 → squad-1.80}/test/ci/test_models.py

@@ -35,7 +35,7 @@ class BackendTestBase(TestCase):
 
     def setUp(self):
         self.group = core_models.Group.objects.create(slug='mygroup')
-        self.project = self.group.projects.create(slug='myproject')
+        self.project = self.group.projects.create(slug='myproject', enabled_plugins_list=['linux-log-parser'])
         self.backend = models.Backend.objects.create()
         self.build = self.project.builds.create(version='1')
 
@@ -430,7 +430,7 @@ class BackendFetchTest(BackendTestBase):
         test_job.refresh_from_db()
         self.assertIsNotNone(test_job.fetched_at)
 
-    @patch.object(models.Backend, '__postprocess_testjob__')
+    @patch('squad.ci.models.Backend.__postprocess_testjob__')
     @patch('squad.ci.backend.null.Backend.job_url', return_value="http://example.com/123")
     @patch('squad.ci.backend.null.Backend.fetch')
     @patch('squad.ci.models.ReceiveTestRun.__call__')

squad-1.79/squad/version.py

@@ -1 +0,0 @@
-__version__ = '1.79'