total-perspective-vortex 3.2.0__tar.gz → 3.3.0__tar.gz

{total_perspective_vortex-3.2.0 → total_perspective_vortex-3.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: total-perspective-vortex
-Version: 3.2.0
+Version: 3.3.0
 Summary: A library for routing entities (jobs, users or groups) to destinations in Galaxy
 Author: Galaxy Contributors
 Maintainer-email: Galaxy Committers <galaxy-committers@lists.galaxyproject.org>
@@ -10,6 +10,7 @@ Project-URL: Documentation, https://total-perspective-vortex.readthedocs.io
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: pydantic>=2
 Requires-Dist: cachetools>=3.1.0
 Requires-Dist: types-cachetools
 Requires-Dist: watchdog
@@ -21,10 +22,14 @@ Requires-Dist: types-requests
 Provides-Extra: cli
 Requires-Dist: total-perspective-vortex; extra == "cli"
 Requires-Dist: galaxy; extra == "cli"
+Provides-Extra: resource-pools
+Requires-Dist: redis>=5.3.0; extra == "resource-pools"
 Provides-Extra: test
 Requires-Dist: total-perspective-vortex[cli]; extra == "test"
+Requires-Dist: total-perspective-vortex[resource-pools]; extra == "test"
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: responses; extra == "test"
+Requires-Dist: fakeredis[lua]; extra == "test"
 Requires-Dist: tox>=2.9.1; extra == "test"
 Requires-Dist: coverage>=4.4.1; extra == "test"
 Provides-Extra: dev

{total_perspective_vortex-3.2.0 → total_perspective_vortex-3.3.0}/pyproject.toml

@@ -14,8 +14,9 @@ authors = [
 maintainers = [
     {name = "Galaxy Committers", email = "galaxy-committers@lists.galaxyproject.org"}
 ]
-version = "3.2.0"
+version = "3.3.0"
 dependencies = [
+    "pydantic>=2",
     "cachetools>=3.1.0",
     "types-cachetools",
     "watchdog",
@@ -32,10 +33,19 @@ cli = [
   "total-perspective-vortex",
   "galaxy"
 ]
+# Required by the Valkey-backed per-user resource pools (see docs: Per-user resource pools).
+# redis-py speaks the Valkey wire protocol.
+resource-pools = [
+  "redis>=5.3.0",
+]
 test = [
   "total-perspective-vortex[cli]",
+  "total-perspective-vortex[resource-pools]",
   "pytest",
   "responses",
+  # Lets the resource-pool store contract test run the real Valkey adapter (Lua script)
+  # without a live server. This is required: the production Lua path must be tested.
+  "fakeredis[lua]",
   "tox>=2.9.1",
   "coverage>=4.4.1",
 ]
@@ -55,6 +65,9 @@ Documentation = "https://total-perspective-vortex.readthedocs.io"
 [tool.setuptools]
 include-package-data = true
 
+[tool.setuptools.packages.find]
+include = ["tpv*"]
+
 [tool.setuptools.package-data]
 "tpv" = ["**/*.j2"]
 

total_perspective_vortex-3.3.0/tests/test_dryrunner_resource_pool.py

@@ -0,0 +1,60 @@
+import os
+import unittest
+
+from tpv.commands.dryrunner import TPVDryRunner
+from tpv.commands.test import mock_galaxy
+from tpv.core.resource_pool import InMemoryAllocationStore
+from tpv.rules import gateway
+
+FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures")
+POOL_CONFIG = os.path.join(FIXTURES, "mapping-resource-pool.yml")
+DEAD_STORE_OVERLAY = os.path.join(FIXTURES, "mapping-resource-pool-dead-store.yml")
+JOB_CONF = os.path.join(FIXTURES, "job_conf_dry_run.yml")
+
+
+class TestMockJob(unittest.TestCase):
+    def test_jobs_get_distinct_default_ids(self):
+        # Pool admission keys the ledger by job id, so a mock job must always have one.
+        a, b = mock_galaxy.Job(), mock_galaxy.Job()
+        self.assertIsInstance(a.id, int)
+        self.assertNotEqual(a.id, b.id)
+
+
+class TestDryRunResourcePools(unittest.TestCase):
+    def _run(self, tool, user, confs, explain=False):
+        runner = TPVDryRunner.from_params(job_conf=JOB_CONF, tool_id=tool, user_email=user, tpv_confs=confs)
+        return runner.run(explain=explain)
+
+    def test_dry_run_maps_a_pooled_config(self):
+        destination, _ = self._run("bigtool", "arthur@vortex.org", [POOL_CONFIG])
+        self.assertEqual(destination.id, "local")
+
+    def test_dry_run_never_touches_the_configured_store(self):
+        # The configured store is a Valkey nothing listens on. If the dry run consulted it, the
+        # job would be deferred fail-closed (or the run would error); an admin's dry run must
+        # neither depend on nor write to production accounting.
+        destination, _ = self._run("bigtool", "arthur@vortex.org", [POOL_CONFIG, DEAD_STORE_OVERLAY])
+        self.assertEqual(destination.id, "local")
+        store = gateway.ACTIVE_DESTINATION_MAPPERS["tpv_dispatcher"].resource_pools.store
+        self.assertIsInstance(store, InMemoryAllocationStore)
+
+    def test_dry_run_still_reports_pools_without_a_store(self):
+        # Swapping in an in-memory store must not hide the misconfiguration the mapper rejects.
+        no_store = os.path.join(FIXTURES, "mapping-resource-pool-no-store.yml")
+        with self.assertRaisesRegex(ValueError, "resource_pool_store"):
+            self._run("bigtool", "arthur@vortex.org", [no_store])
+
+    def test_explain_reports_which_pools_govern_the_job_and_why(self):
+        _, collector = self._run("bigtool", "arthur@vortex.org", [POOL_CONFIG], explain=True)
+        trace = collector.render()
+        self.assertIn("Resource Pools", trace)
+        # bigtool is 64 cores against a 32-core budget: governed by 'default', classified oversize.
+        self.assertIn("default", trace)
+        self.assertIn("oversize", trace.lower())
+        self.assertIn("32", trace)
+
+    def test_explain_reports_a_permanently_rejected_request(self):
+        # hugetool (200 cores) exceeds the oversize ceiling and can never be scheduled.
+        destination, collector = self._run("hugetool", "arthur@vortex.org", [POOL_CONFIG], explain=True)
+        self.assertIsNone(destination)
+        self.assertIn("can never be scheduled", collector.render())

{total_perspective_vortex-3.2.0 → total_perspective_vortex-3.3.0}/tests/test_entity.py

@@ -70,7 +70,7 @@ class TestEntity(unittest.TestCase):
         # create a destination
         destination = loader.config.destinations["k8s_environment"]
         # serialize the destination
-        serialized_destination = destination.dict()
+        serialized_destination = destination.model_dump()
         # deserialize the same destination
         deserialized_destination = Destination(evaluator=loader, **serialized_destination)
         # make sure the deserialized destination is the same as the original
@@ -83,7 +83,7 @@ class TestEntity(unittest.TestCase):
         # create a tool
         tool = loader.config.tools["limbo"]
         # serialize the tool
-        serialized_tool = tool.dict()
+        serialized_tool = tool.model_dump()
         # deserialize the same tool
         deserialized_tool = Tool(evaluator=loader, **serialized_tool)
         # make sure the deserialized tool is the same as the original

{total_perspective_vortex-3.2.0 → total_perspective_vortex-3.3.0}/tests/test_explain.py

@@ -3,11 +3,14 @@ import unittest
 from unittest.mock import MagicMock
 
 import yaml
+from galaxy.jobs import JobDestination
 from galaxy.jobs.mapper import JobMappingException
 
 from tpv.commands.dryrunner import TPVDryRunner
+from tpv.commands.test import mock_galaxy
 from tpv.core.entities import SchedulingTags
 from tpv.core.explain import ExplainCollector, ExplainPhase
+from tpv.rules import gateway
 
 
 class TestExplainCollectorUnit(unittest.TestCase):
@@ -496,3 +499,59 @@ class TestDryRunExplain(unittest.TestCase):
             tpv_confs=[self._fixture_path("mapping-basic.yml")],
         )
         self.assertIsNone(runner.tool)
+
+
+
+class TestGatewayExplainOnFailure(unittest.TestCase):
+    """Tests for the tpv_explain_on_failure gateway config option."""
+
+    @staticmethod
+    def _fixture_path(name):
+        return os.path.join(os.path.dirname(__file__), f"fixtures/{name}")
+
+    def _call_gateway(self, tool_id, referrer=None, explain_collector=None):
+        gateway.ACTIVE_DESTINATION_MAPPERS = {}
+        app = mock_galaxy.App(job_conf=self._fixture_path("job_conf.yml"))
+        tool = mock_galaxy.Tool(tool_id)
+        job = mock_galaxy.Job()
+        user = mock_galaxy.User("gargravarr", "fairycake@vortex.org")
+        return gateway.map_tool_to_destination(
+            app, job, tool, user,
+            referrer=referrer,
+            tpv_config_files=[self._fixture_path("mapping-basic.yml")],
+            explain_collector=explain_collector,
+        )
+
+    def test_no_log_when_param_not_set(self):
+        """No warning is logged on failure when tpv_explain_on_failure is not configured."""
+        with self.assertNoLogs("tpv.rules.gateway", level="WARNING"):
+            with self.assertRaises(JobMappingException):
+                self._call_gateway("unschedulable_tool")
+
+    def test_logs_trace_on_mapping_failure(self):
+        """A WARNING containing the scheduling trace is logged when tpv_explain_on_failure=true and mapping fails."""
+        referrer = JobDestination(id="tpv_dispatcher", params={"tpv_explain_on_failure": True})
+        with self.assertLogs("tpv.rules.gateway", level="WARNING") as cm:
+            with self.assertRaises(JobMappingException):
+                self._call_gateway("unschedulable_tool", referrer=referrer)
+        self.assertEqual(len(cm.records), 1)
+        trace = cm.records[0].getMessage()
+        self.assertIn("TPV SCHEDULING DECISION TRACE", trace)
+        self.assertIn("REJECTED", trace)
+        self.assertIn("tag mismatch", trace)
+        self.assertIn("No destinations", trace)
+
+    def test_no_log_on_successful_mapping(self):
+        """No warning is logged when tpv_explain_on_failure=true but mapping succeeds."""
+        referrer = JobDestination(id="tpv_dispatcher", params={"tpv_explain_on_failure": True})
+        with self.assertNoLogs("tpv.rules.gateway", level="WARNING"):
+            destination = self._call_gateway("bwa", referrer=referrer)
+        self.assertIsNotNone(destination)
+
+    def test_explicit_collector_not_logged_by_gateway(self):
+        """When an explicit explain_collector is passed (dry-run path), the gateway does not log on failure."""
+        collector = ExplainCollector()
+        with self.assertNoLogs("tpv.rules.gateway", level="WARNING"):
+            with self.assertRaises(JobMappingException):
+                self._call_gateway("unschedulable_tool", explain_collector=collector)
+        self.assertTrue(len(collector.steps) > 0)

total_perspective_vortex-3.3.0/tests/test_helpers.py

@@ -0,0 +1,265 @@
+"""Unit tests module for the helper functions"""
+
+import unittest
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from tpv.commands.test import mock_galaxy
+from tpv.core.helpers import (
+    get_dataset_attributes,
+    get_input_dataset,
+    get_input_datasets,
+    get_input_size,
+    input_size,
+    weighted_choice,
+    weighted_random_sampling,
+)
+
+
+class TestHelpers(unittest.TestCase):
+    """Tests for helper functions"""
+
+    def test_get_dataset_attributes(self):
+        """Test that the function returns a dictionary with the correct attributes"""
+        job = mock_galaxy.Job()
+        dataset = mock_galaxy.Dataset("test.txt", file_size=7 * 1024**3, object_store_id="files1")
+        job.add_input_dataset(mock_galaxy.DatasetAssociation("test", dataset))
+        dataset_attributes = get_dataset_attributes(job.input_datasets)
+        expected_result = {dataset.id: {"object_store_id": "files1", "size": 7 * 1024**3}}
+        self.assertEqual(dataset_attributes, expected_result)
+
+    @staticmethod
+    def _job_with_multiple_data_param():
+        """A job with a `multiple="true"` data param `inputs` holding two datasets.
+
+        Galaxy records the first dataset twice: once as `inputs` (the copy-metadata alias for
+        the first element) and once as `inputs1`.
+        """
+        job = mock_galaxy.Job()
+        first = mock_galaxy.DatasetAssociation(
+            "first", mock_galaxy.Dataset("first.txt", file_size=3 * 1024**3), extension="txt"
+        )
+        second = mock_galaxy.DatasetAssociation(
+            "second", mock_galaxy.Dataset("second.txt", file_size=5 * 1024**3), extension="txt"
+        )
+        job.add_input_dataset(first, name="inputs")
+        job.add_input_dataset(first, name="inputs1")
+        job.add_input_dataset(second, name="inputs2")
+        return job
+
+    def test_get_input_datasets_deduplicates_multiple_data_param_alias(self):
+        """The `name`/`name1` alias of a multiple data param must not yield the dataset twice"""
+        job = self._job_with_multiple_data_param()
+        datasets = get_input_datasets(job, "inputs")
+        self.assertEqual([dataset.name for dataset in datasets], ["first", "second"])
+
+    def test_get_input_datasets_matches_collection_element_names(self):
+        """A collection param is recorded as `name1`..`nameN`, with no unsuffixed entry"""
+        job = mock_galaxy.Job()
+        for index in range(1, 4):
+            job.add_input_dataset(
+                mock_galaxy.DatasetAssociation(
+                    f"element{index}", mock_galaxy.Dataset(f"element{index}.txt", file_size=1 * 1024**3)
+                ),
+                name=f"inputs{index}",
+            )
+        datasets = get_input_datasets(job, "inputs")
+        self.assertEqual([dataset.name for dataset in datasets], ["element1", "element2", "element3"])
+
+    def test_get_input_datasets_ignores_other_params(self):
+        """Only datasets recorded for the requested param are returned"""
+        job = self._job_with_multiple_data_param()
+        job.add_input_dataset(
+            mock_galaxy.DatasetAssociation("ref", mock_galaxy.Dataset("ref.txt", file_size=9 * 1024**3)),
+            name="reference",
+        )
+        self.assertEqual([dataset.name for dataset in get_input_datasets(job, "reference")], ["ref"])
+        self.assertEqual([dataset.name for dataset in get_input_datasets(job, "inputs")], ["first", "second"])
+        self.assertEqual([dataset.name for dataset in get_input_datasets(job)], ["first", "second", "ref"])
+
+    def test_get_input_datasets_skips_unset_optional_params(self):
+        """An unset optional data param is recorded with no dataset at all"""
+        job = mock_galaxy.Job()
+        job.input_datasets.append(mock_galaxy.JobToInputDatasetAssociation("inputs", None))
+        self.assertEqual(get_input_datasets(job, "inputs"), [])
+        self.assertIsNone(get_input_dataset(job, "inputs"))
+        self.assertEqual(get_input_size(job, "inputs"), 0)
+
+    def test_get_input_dataset_returns_first_match(self):
+        job = self._job_with_multiple_data_param()
+        dataset = get_input_dataset(job, "inputs")
+        self.assertEqual(dataset.name, "first")
+        self.assertIsNone(get_input_dataset(job, "nonexistent"))
+
+    def test_get_input_size_totals_all_inputs_by_default(self):
+        job = self._job_with_multiple_data_param()
+        self.assertEqual(get_input_size(job), 8)
+        # the existing input_size helper deduplicates the same way
+        self.assertEqual(input_size(job), 8)
+
+    def test_get_input_size_by_param_name(self):
+        job = self._job_with_multiple_data_param()
+        job.add_input_dataset(
+            mock_galaxy.DatasetAssociation("ref", mock_galaxy.Dataset("ref.txt", file_size=9 * 1024**3)),
+            name="reference",
+        )
+        self.assertEqual(get_input_size(job, "inputs"), 8)
+        self.assertEqual(get_input_size(job, "reference"), 9)
+        self.assertEqual(get_input_size(job), 17)
+
+    def test_get_input_size_adjusts_compressed_inputs(self):
+        job = mock_galaxy.Job()
+        job.add_input_dataset(
+            mock_galaxy.DatasetAssociation(
+                "compressed",
+                mock_galaxy.Dataset("compressed.fastq.gz", file_size=2 * 1024**3),
+                extension="fastqsanger.gz",
+            ),
+            name="inputs1",
+        )
+        job.add_input_dataset(
+            mock_galaxy.DatasetAssociation(
+                "uncompressed",
+                mock_galaxy.Dataset("uncompressed.fastq", file_size=4 * 1024**3),
+                extension="fastqsanger",
+            ),
+            name="inputs2",
+        )
+        self.assertAlmostEqual(get_input_size(job, "inputs"), 2 * 3.4 + 4)
+        self.assertAlmostEqual(get_input_size(job, "inputs", compression_factor=2), 2 * 2 + 4)
+        self.assertEqual(get_input_size(job, "inputs", estimate_uncompressed_size=False), 6)
+
+    def test_weighted_random_sampling_without_weights_uses_unweighted_sampling(self):
+        """When no destination defines params.weight, use unweighted random sampling."""
+        destinations = [
+            SimpleNamespace(id="dest_a", params={}),
+            SimpleNamespace(id="dest_b", params=None),
+            SimpleNamespace(id="dest_c", params={"foo": "bar"}),
+        ]
+        sampled_destinations = [destinations[2], destinations[0], destinations[1]]
+
+        with patch("tpv.core.helpers.random.sample", return_value=sampled_destinations) as sample_mock:
+            with patch("tpv.core.helpers.random.choices") as choices_mock:
+                result = weighted_random_sampling(destinations)
+
+        self.assertEqual(result, sampled_destinations)
+        sample_mock.assert_called_once_with(destinations, k=3)
+        choices_mock.assert_not_called()
+
+    def test_weighted_random_sampling_with_weights_uses_weighted_choices(self):
+        """When any destination defines params.weight, use weighted random choices."""
+        destinations = [
+            SimpleNamespace(id="dest_a", params={"weight": 5}),
+            SimpleNamespace(id="dest_b", params={}),
+            SimpleNamespace(id="dest_c", params=None),
+        ]
+        sampled_destinations = [destinations[0], destinations[0], destinations[2]]
+
+        with patch("tpv.core.helpers.random.choices", return_value=sampled_destinations) as choices_mock:
+            with patch("tpv.core.helpers.random.sample") as sample_mock:
+                result = weighted_random_sampling(destinations)
+
+        self.assertEqual(result, sampled_destinations)
+        choices_mock.assert_called_once_with(destinations, weights=[5, 1, 1], k=3)
+        sample_mock.assert_not_called()
+
+    def test_weighted_choice_without_weights_uses_unweighted_choice(self):
+        """When no item defines weight, use unweighted random choice."""
+        items = [
+            {"value": "/fast/jobs"},
+            {"value": "/slow/jobs"},
+            {"value": "/backup/jobs", "foo": "bar"},
+        ]
+
+        with patch("tpv.core.helpers.random.sample", return_value=[items[1]]) as sample_mock:
+            with patch("tpv.core.helpers.random.choices") as choices_mock:
+                result = weighted_choice(items)
+
+        self.assertEqual(result, items[1])
+        sample_mock.assert_called_once_with(items, k=1)
+        choices_mock.assert_not_called()
+
+    def test_weighted_choice_with_weights_uses_weighted_choices(self):
+        """When any item defines weight, use weighted random choices."""
+        items = [
+            {"value": "/fast/jobs", "weight": 3},
+            {"value": "/slow/jobs"},
+            {"value": "/backup/jobs"},
+        ]
+        with patch("tpv.core.helpers.random.choices", return_value=[items[0]]) as choices_mock:
+            with patch("tpv.core.helpers.random.sample") as sample_mock:
+                result = weighted_choice(items)
+
+        self.assertEqual(result, items[0])
+        choices_mock.assert_called_once_with(items, weights=[3, 1, 1], k=1)
+        sample_mock.assert_not_called()
+
+    def test_weighted_choice_missing_weight_defaults_to_one(self):
+        """Items without a weight key should default to weight 1."""
+        items = [
+            {"value": "/a", "weight": 5},
+            {"value": "/b"},
+            {"value": "/c"},
+        ]
+        with patch("tpv.core.helpers.random.choices", return_value=[items[1]]) as choices_mock:
+            result = weighted_choice(items)
+
+        self.assertEqual(result, items[1])
+        choices_mock.assert_called_once_with(items, weights=[5, 1, 1], k=1)
+
+    def test_weighted_choice_zero_weights_falls_back_to_unweighted(self):
+        """If all weights are zero or negative, fall back to unweighted selection."""
+        items = [
+            {"value": "/drained/jobs", "weight": 0},
+            {"value": "/also-drained/jobs", "weight": -1},
+        ]
+        with patch("tpv.core.helpers.random.sample", return_value=[items[0]]) as sample_mock:
+            with patch("tpv.core.helpers.random.choices") as choices_mock:
+                result = weighted_choice(items)
+
+        self.assertEqual(result, items[0])
+        sample_mock.assert_called_once_with(items, k=1)
+        choices_mock.assert_not_called()
+
+    def test_weighted_choice_negative_weight_is_clamped_to_zero(self):
+        """A negative weight is clamped to 0 while positive weights are preserved."""
+        items = [
+            {"value": "/a", "weight": 5},
+            {"value": "/b", "weight": -3},
+        ]
+        with patch("tpv.core.helpers.random.choices", return_value=[items[0]]) as choices_mock:
+            with patch("tpv.core.helpers.random.sample") as sample_mock:
+                weighted_choice(items)
+
+        choices_mock.assert_called_once_with(items, weights=[5, 0], k=1)
+        sample_mock.assert_not_called()
+
+    def test_weighted_choice_default_weight_keys_use_unweighted(self):
+        """If every item has weight: 1 (the default), use unweighted choice."""
+        items = [
+            {"value": "/a", "weight": 1},
+            {"value": "/b", "weight": 1},
+        ]
+        with patch("tpv.core.helpers.random.sample", return_value=[items[0]]) as sample_mock:
+            with patch("tpv.core.helpers.random.choices") as choices_mock:
+                weighted_choice(items)
+
+        sample_mock.assert_called_once_with(items, k=1)
+        choices_mock.assert_not_called()
+
+    def test_weighted_choice_empty_list_raises_value_error(self):
+        """An empty list should raise ValueError."""
+        with self.assertRaises(ValueError):
+            weighted_choice([])
+
+    def test_weighted_choice_returns_selected_item(self):
+        """The helper should return the selected item, not a derived string value."""
+        items = [
+            {"value": "/primary/jobs", "weight": 10},
+            {"value": "/secondary/jobs", "weight": 1},
+        ]
+        with patch("tpv.core.helpers.random.choices", return_value=[items[1]]):
+            result = weighted_choice(items)
+
+        self.assertIsInstance(result, dict)
+        self.assertEqual(result, items[1])

{total_perspective_vortex-3.2.0 → total_perspective_vortex-3.3.0}/tests/test_mapper_context.py

@@ -44,7 +44,7 @@ class TestMapperContext(unittest.TestCase):
         self.assertEqual(destination.params["native_spec"], "--mem 15 --cores 5 --gpus 4")
 
     def test_context_variable_overridden_in_rule(self):
-        # test that job will not fail with 40GB input size because large_input_size has been set to 60
+        # test that job will not fail with 40GiB input size because large_input_size has been set to 60
         tool = mock_galaxy.Tool("bwa")
         user = mock_galaxy.User("gargravarr", "fairycake@vortex.org")
         datasets = [mock_galaxy.DatasetAssociation("test", mock_galaxy.Dataset("test.txt", file_size=40 * 1024**3))]

{total_perspective_vortex-3.2.0 → total_perspective_vortex-3.3.0}/tests/test_mapper_params_specific.py

@@ -61,12 +61,15 @@ class TestParamsSpecific(unittest.TestCase):
         user = mock_galaxy.User("trillian", "panic@vortex.org")
 
         destination = self._map_to_destination(tool, user)
+        # Galaxy's resubmit state handler reads the target destination from the
+        # "environment" key, so TPV's user-facing "destination" key must be
+        # translated when building the Galaxy JobDestination.
         self.assertEqual(
             destination.resubmit,
             [
                 {
                     "condition": "memory_limit_reached and attempt <= 3",
-                    "destination": "tpv_dispatcher",
+                    "environment": "tpv_dispatcher",
                     "delay": "attempt * 30",
                 }
             ],