insightconnect-plugin-runtime 5.4.0__py3-none-any.whl → 5.4.2__py3-none-any.whl

insightconnect_plugin_runtime/server.py

@@ -6,8 +6,6 @@ import logging
 from apispec import APISpec
 from apispec.ext.marshmallow import MarshmallowPlugin
 from apispec_webframeworks.flask import FlaskPlugin
-from apscheduler.schedulers.background import BackgroundScheduler
-
 
 from flask import Flask, request_started, request
 import gunicorn.app.base
@@ -17,6 +15,7 @@ import structlog
 from pythonjsonlogger.jsonlogger import JsonFormatter
 from requests import get as request_get
 from requests.exceptions import HTTPError, MissingSchema, Timeout, JSONDecodeError
+from time import sleep
 from werkzeug.utils import secure_filename
 
 from insightconnect_plugin_runtime.api.schemas import (
@@ -42,6 +41,8 @@ API_TITLE = "InsightConnect Plugin Runtime API"
 API_VERSION = "1.0"
 OPEN_API_VERSION = "2.0"
 VERSION_MAPPING = {"legacy": "/", "v1": "/api/v1"}
+CPS_RETRY = 5
+RETRY_SLEEP = 10
 CPS_ENDPOINT = os.getenv("CPS_ENDPOINT")  # endpoint to retrieve plugin custom configs (set in cloud deployments.tf)
 DEFAULT_SCHEDULE_INTERVAL_MINUTES = 3  # default to 3 as most integrations run on 5 minute interval
 
@@ -105,7 +106,7 @@ class PluginServer(gunicorn.app.base.BaseApplication):
         self.workers = workers
         self.threads = threads
         # initialise before reaching out to CPS for configured values
-        self.config_options, self.schedule_interval = {}, None
+        self.config_options = {}
         self.get_plugin_properties_from_cps()
         self.app, self.blueprints = self.create_flask_app()
 
@@ -183,25 +184,28 @@ class PluginServer(gunicorn.app.base.BaseApplication):
     def get_plugin_properties_from_cps(self):
         # Call out to komand-props to get configurations related to only the plugin pod running.
         if is_running_in_cloud() and self.plugin.tasks:
-            self.logger.info("Getting plugin configuration information...")
-            try:
-                request_response = request_get(CPS_ENDPOINT, timeout=30)
-                resp_json = request_response.json()
-                plugin = self.plugin.name.lower().replace(" ", "_")  # match how we name our images
-                plugin_config = resp_json.get("plugins", {}).get(plugin, {})
-
-                self.config_options = plugin_config
-                self.schedule_interval = resp_json.get("config", {}).get("interval", DEFAULT_SCHEDULE_INTERVAL_MINUTES)
-            except MissingSchema as missing_schema:
-                self.logger.error(f"Invalid URL being requested: {CPS_ENDPOINT}, error={missing_schema}")
-            except Timeout as timeout:
-                self.logger.error(f"Connection timeout hit. CPS={CPS_ENDPOINT}, error={timeout}")
-            except HTTPError as http_error:
-                self.logger.error(f"Connection error when trying to reach CPS. CPS={CPS_ENDPOINT}, error={http_error}")
-            except JSONDecodeError as bad_json:
-                self.logger.error(f"Got bad JSON back. Response content={request_response.content}, error={bad_json}")
-            except Exception as error:
-                self.logger.error(f"Hit an unexpected error when retrieving plugin custom configs, error={error}")
+            for attempt in range(1, CPS_RETRY + 1):
+                self.logger.info(f"Getting plugin configuration information... (attempt {attempt}/{CPS_RETRY})")
+                try:
+                    request_response = request_get(CPS_ENDPOINT, timeout=30)
+                    resp_json = request_response.json()
+                    plugin = self.plugin.name.lower().replace(" ", "_")  # match how we name our images
+                    plugin_config = resp_json.get("plugins", {}).get(plugin, {})
+
+                    self.config_options = plugin_config
+                    self.logger.info("Plugin configuration successfully retrieved...")
+                    break
+                except MissingSchema as missing_schema:
+                    self.logger.error(f"Invalid URL being requested: {CPS_ENDPOINT}, error={missing_schema}")
+                except Timeout as timeout:
+                    self.logger.error(f"Connection timeout hit. CPS={CPS_ENDPOINT}, error={timeout}")
+                except HTTPError as http_error:
+                    self.logger.error(f"Connection error when trying to reach CPS. CPS={CPS_ENDPOINT}, error={http_error}")
+                except JSONDecodeError as bad_json:
+                    self.logger.error(f"Got bad JSON back. Response content={request_response.content}, error={bad_json}")
+                except Exception as error:
+                    self.logger.error(f"Hit an unexpected error when retrieving plugin custom configs, error={error}")
+                sleep(RETRY_SLEEP)
 
     def register_api_spec(self):
         """Register all swagger schema definitions and path objects"""
@@ -253,21 +257,6 @@ class PluginServer(gunicorn.app.base.BaseApplication):
                 blueprint, url_prefix=VERSION_MAPPING[blueprint.name]
             )
 
-    def register_scheduled_tasks(self):
-        # Once Flask server is up and running also start the get_plugin_configs
-        try:
-            if self.plugin.tasks and self.schedule_interval:
-                self.logger.info(f"Starting scheduled task(s) to run at interval of {self.schedule_interval} minutes...")
-                scheduler = BackgroundScheduler(daemon=True)
-                scheduler.add_job(self.get_plugin_properties_from_cps, 'interval', minutes=self.schedule_interval)
-                scheduler.start()
-            else:
-                reason = "No tasks found found within plugin," if self.schedule_interval else "No schedule defined,"
-                self.logger.info(f"{reason},not starting scheduled tasks...")
-        except Exception as exception:
-            self.logger.error("Unable to start up scheduler, plugin will not be refreshing these configuration values."
-                              f"Error={exception}", exc_info=True)
-
     @staticmethod
     def bind_request_details(sender: Flask, **extras) -> None:
         """
@@ -299,8 +288,6 @@ class PluginServer(gunicorn.app.base.BaseApplication):
             try:
                 self.register_blueprint()
                 self.register_api_spec()
-                if is_running_in_cloud():
-                    self.register_scheduled_tasks()
                 self.arbiter.run()
             except RuntimeError as e:
                 sys.stderr.write("\nError: %s\n" % e)

{insightconnect_plugin_runtime-5.4.0.dist-info → insightconnect_plugin_runtime-5.4.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: insightconnect-plugin-runtime
-Version: 5.4.0
+Version: 5.4.2
 Summary: InsightConnect Plugin Runtime
 Home-page: https://github.com/rapid7/komand-plugin-sdk-python
 Author: Rapid7 Integrations Alliance
@@ -26,7 +26,6 @@ Requires-Dist: apispec-webframeworks ==1.0.0
 Requires-Dist: blinker ==1.7.0
 Requires-Dist: structlog ==24.1.0
 Requires-Dist: python-json-logger ==2.0.7
-Requires-Dist: APScheduler ==3.10.4
 
 
 # InsightConnect Python Plugin Runtime ![Build Status](https://github.com/rapid7/komand-plugin-sdk-python/workflows/Continuous%20Integration/badge.svg)
@@ -186,7 +185,7 @@ Running a specific test file:
 |                   | Plugin  | Slim Plugin |
 |:------------------|:-------:|:-----------:|
 | Python Version    | 3.9.18  |   3.9.18    |
-| OS                | Alpine  |  Bookworm   |
+| OS                | Alpine  |  Bullseye   |
 | Package installer |   apk   |     apt     |
 | Shell             | /bin/sh |  /bin/bash  |
 | Image Size        | ~350MB  |   ~180MB    |
@@ -213,6 +212,8 @@ after cloning this repository.
 
 ## Changelog
 
+* 5.4.2 - Remove background scheduler to simplify when the server gets custom_config values | Revert to use `bullseye` for slim image.
+* 5.4.1 - Retry logic added to get values from external API for custom_config values.
 * 5.4.0 - Implementation of custom_config parameter for plugin tasks | Alpine image updated OpenSSL and expat | Use `bookworm` for slim image.
 * 5.3.2 - Updated OpenSSL in alpine image and core packages to latest versions.
 * 5.3.1 - New logging added to the beginning and end of a task | New logging when an exception is instantiated.

{insightconnect_plugin_runtime-5.4.0.dist-info → insightconnect_plugin_runtime-5.4.2.dist-info}/RECORD

@@ -8,7 +8,7 @@ insightconnect_plugin_runtime/helper.py,sha256=m5PxN04-NPXM1X10S2wwjqmiLvnNntd6T
 insightconnect_plugin_runtime/metrics.py,sha256=hf_Aoufip_s4k4o8Gtzz90ymZthkaT2e5sXh5B4LcF0,3186
 insightconnect_plugin_runtime/plugin.py,sha256=A6GMrYTNFpyCeEP00jRbp89Xu6KdQFq0HOVWV2AxA6Q,22969
 insightconnect_plugin_runtime/schema.py,sha256=jTNc6KAMqFpaDVWrAYhkVC6e8I63P3X7uVlJkAr1hiY,583
-insightconnect_plugin_runtime/server.py,sha256=2TMIf5lXmYvDgQu2a6VOGcu6saJmcQoDTKfNNvyBj9E,12693
+insightconnect_plugin_runtime/server.py,sha256=vm0fJcnaI9Z3WYB37wUFkOjcym_kMlSm3lAyFP3ARiM,11771
 insightconnect_plugin_runtime/step.py,sha256=KdERg-789-s99IEKN61DR08naz-YPxyinPT0C_T81C4,855
 insightconnect_plugin_runtime/task.py,sha256=d-H1EAzVnmSdDEJtXyIK5JySprxpF9cetVoFGtWlHrg,123
 insightconnect_plugin_runtime/trigger.py,sha256=Zq3cy68N3QxAGbNZKCID6CZF05Zi7YD2sdy_qbedUY8,874
@@ -73,12 +73,12 @@ tests/unit/test_metrics.py,sha256=PjjTrB9w7uQ2Q5UN-893-SsH3EGJuBseOMHSD1I004s,79
 tests/unit/test_oauth.py,sha256=nbFG0JH1x04ExXqSe-b5BGdt_hJs7DP17eUa6bQzcYI,2093
 tests/unit/test_plugin.py,sha256=ZTNAZWwZhDIAbxkVuWhnz9FzmojbijgMmsLWM2mXQI0,4160
 tests/unit/test_schema.py,sha256=swWZPRo_Q4M6VHte-srmxcV2wH-XS7pgmNRxpaL0Qrg,642
-tests/unit/test_server_cloud_plugins.py,sha256=I4ejyCP8zK0H0QD8KG3LHuOlKRXFGORbL0BOrVqyzME,5515
+tests/unit/test_server_cloud_plugins.py,sha256=GvJCOgpaxw3jOz_ABCkui_Ymai0kAZK1MqR0ldERnhw,4544
 tests/unit/test_server_spec.py,sha256=je97BaktgK0Fiz3AwFPkcmHzYtOJJNqJV_Fw5hrvqX4,644
 tests/unit/test_trigger.py,sha256=E53mAUoVyponWu_4IQZ0IC1gQ9lakBnTn_9vKN2IZfg,1692
 tests/unit/test_variables.py,sha256=OUEOqGYZA3Nd5oKk5GVY3hcrWKHpZpxysBJcO_v5gzs,291
 tests/unit/utils.py,sha256=-GB3nz4YJ2nMm0AR7Fm4lCNJMHCWOmLtNI1N4b9hRdo,470
-insightconnect_plugin_runtime-5.4.0.dist-info/METADATA,sha256=uNV5HcyCbc49hSsQIY3UfJ2GnbB0ktGamEn03NcNJyA,12287
-insightconnect_plugin_runtime-5.4.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-insightconnect_plugin_runtime-5.4.0.dist-info/top_level.txt,sha256=AJtyJOpiFzHxsbHUICTcUKXyrGQ3tZxhrEHsPjJBvEA,36
-insightconnect_plugin_runtime-5.4.0.dist-info/RECORD,,
+insightconnect_plugin_runtime-5.4.2.dist-info/METADATA,sha256=8h98fhwRRCE9c7y4U6-96Qw7ci0h89KqvnrS2r-XPg0,12472
+insightconnect_plugin_runtime-5.4.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+insightconnect_plugin_runtime-5.4.2.dist-info/top_level.txt,sha256=AJtyJOpiFzHxsbHUICTcUKXyrGQ3tZxhrEHsPjJBvEA,36
+insightconnect_plugin_runtime-5.4.2.dist-info/RECORD,,

tests/unit/test_server_cloud_plugins.py

@@ -1,9 +1,9 @@
 from requests.exceptions import HTTPError, Timeout, TooManyRedirects
 from parameterized import parameterized
-from unittest import TestCase
+from unittest import TestCase, skip
 from unittest.mock import patch, MagicMock
 
-from insightconnect_plugin_runtime.server import PluginServer, DEFAULT_SCHEDULE_INTERVAL_MINUTES
+from insightconnect_plugin_runtime.server import PluginServer
 from tests.plugin.hello_world import KomandHelloWorld
 from .utils import MockResponse, Logger
 
@@ -18,26 +18,20 @@ class TestServerCloudPlugins(TestCase):
         self.plugin_name = self.plugin.name.lower().replace(" ", "_")
 
     @parameterized.expand([["Set cloud to false", False], ["Set cloud to true", True]])
-    @patch("insightconnect_plugin_runtime.server.PluginServer.register_scheduled_tasks")
     @patch("insightconnect_plugin_runtime.server.request_get")
-    def test_cloud_plugin_no_tasks_ignore_cps(self, _test_name, cloud, mocked_req, mocked_scheduler, mock_cloud, _run):
+    def test_cloud_plugin_no_tasks_ignore_cps(self, _test_name, cloud, mocked_req, mock_cloud, _run):
         mock_cloud.return_value = cloud  # Mock plugin running in cloud vs not
         self.plugin.tasks = None  # ensure still no tasks as other tests edit this and could fail before reverting
         plugin_server = PluginServer(self.plugin)  # this plugin has no tasks by default
 
         plugin_server.start()
         self.assertEqual(plugin_server.config_options, {})
-        self.assertEqual(plugin_server.schedule_interval, None)
-
-        # Depending on if we're running cloud or not this could be called but we only reach to CPS if we have tasks
-        self.assertEquals(mocked_scheduler.called, cloud)
 
         # Plugin server never calls out to CPS as either we are not running in cloud mode or have no tasks.
         self.assertFalse(mocked_req.called)
 
-    @patch("insightconnect_plugin_runtime.server.PluginServer.register_scheduled_tasks")
     @patch("insightconnect_plugin_runtime.server.request_get")
-    def test_cloud_plugin_calls_cps(self, mocked_req, mocked_scheduler, _mock_cloud, _run):
+    def test_cloud_plugin_calls_cps(self, mocked_req, _mock_cloud, _run):
         mocked_req.return_value = MockResponse({"plugins":{self.plugin_name: PLUGIN_VALUE_1, 'plugin': PLUGIN_VALUE_2},
                                                 "config": {"interval": 25}})
         self.plugin.tasks = 'fake tasks'  # this plugin by default has no tasks so force it to have some
@@ -49,17 +43,15 @@ class TestServerCloudPlugins(TestCase):
 
         # We only save the plugin config for the current config and ignore `other_plugin`
         self.assertDictEqual(plugin_server.config_options, PLUGIN_VALUE_1)
-        self.assertEqual(plugin_server.schedule_interval, SCHEDULE_INTERVAL)
 
-        # We should now schedule this to run
-        self.assertEquals(mocked_scheduler.called, True)
         self.plugin.tasks = None  # reset tasks value
 
     @parameterized.expand([["error", HTTPError], ["error", Timeout], ["unexpected", TooManyRedirects]])
-    @patch("insightconnect_plugin_runtime.server.PluginServer.register_scheduled_tasks")
     @patch("insightconnect_plugin_runtime.server.request_get")
     @patch("structlog.get_logger")
-    def test_cps_schedule_raises_an_error(self, test_cond, exception, log, mocked_req, _mocked_scheduler, _mock_cloud, _run):
+    @patch("insightconnect_plugin_runtime.server.CPS_RETRY", new=2)  # reduce retries in unit tests
+    @patch("insightconnect_plugin_runtime.server.RETRY_SLEEP", new=1)  # reduce sleep in unit tests
+    def test_cps_raises_an_error(self, test_cond, exception, log, mocked_req, _mock_cloud, _run):
         log.return_value = Logger()
         # If we have successfully got config and scheduler options, and later this call fails we should keep values
         mocked_req.return_value = MockResponse({"plugins": {self.plugin_name: PLUGIN_VALUE_1, 'plugin': PLUGIN_VALUE_2},
@@ -70,9 +62,8 @@ class TestServerCloudPlugins(TestCase):
         plugin_server.start()
 
         self.assertDictEqual(plugin_server.config_options, PLUGIN_VALUE_2)
-        self.assertEqual(plugin_server.schedule_interval, DEFAULT_SCHEDULE_INTERVAL_MINUTES)  # no resp uses default
 
-        # First call has happened and now successful - force scheduler to hit specific handled and unexpected errors.
+        # First call has happened and now successful - force to hit specific handled and unexpected errors.
         mocked_req.side_effect = exception("Warning HTTP error returned...")
         plugin_server.get_plugin_properties_from_cps()
         # we log error in all and `unexpected` in TooManyRedirects as there is no direct catch for this
@@ -80,16 +71,13 @@ class TestServerCloudPlugins(TestCase):
 
         # Values should not have changed
         self.assertDictEqual(plugin_server.config_options, PLUGIN_VALUE_2)
-        self.assertEqual(plugin_server.schedule_interval, DEFAULT_SCHEDULE_INTERVAL_MINUTES)
 
-        # Next schedule returns updated values, new schedule and no configurations for plugins
-        new_schedule = 12345678
-        mocked_req.return_value = MockResponse({"config": {"interval": new_schedule}})
+        # Next schedule returns no configurations for plugins
+        mocked_req.return_value = MockResponse({})
         mocked_req.side_effect = None
         plugin_server.get_plugin_properties_from_cps()
 
         # And this new values are now updated for the plugin server
         self.assertDictEqual(plugin_server.config_options, {})
-        self.assertEqual(plugin_server.schedule_interval, new_schedule)
 
         self.plugin.tasks = None  # reset tasks value