canvas 0.2.11__py3-none-any.whl → 0.3.1__py3-none-any.whl

plugin_runner/plugin_runner.py

@@ -0,0 +1,338 @@
+import asyncio
+import importlib.util
+import json
+import os
+import pathlib
+import signal
+import sys
+import time
+import traceback
+from collections import defaultdict
+from types import FrameType
+from typing import Any, Optional
+
+import grpc
+import statsd
+
+from canvas_generated.messages.plugins_pb2 import (
+    ReloadPluginsRequest,
+    ReloadPluginsResponse,
+)
+from canvas_generated.services.plugin_runner_pb2_grpc import (
+    PluginRunnerServicer,
+    add_PluginRunnerServicer_to_server,
+)
+from canvas_sdk.effects import Effect
+from canvas_sdk.events import Event, EventResponse, EventType
+from canvas_sdk.protocols import ClinicalQualityMeasure
+from canvas_sdk.utils.stats import get_duration_ms, tags_to_line_protocol
+from logger import log
+from plugin_runner.authentication import token_for_plugin
+from plugin_runner.plugin_synchronizer import publish_message
+from plugin_runner.sandbox import Sandbox
+
+ENV = os.getenv("ENV", "development")
+
+IS_PRODUCTION = ENV == "production"
+
+MANIFEST_FILE_NAME = "CANVAS_MANIFEST.json"
+
+SECRETS_FILE_NAME = "SECRETS.json"
+
+# specify a local plugin directory for development
+PLUGIN_DIRECTORY = "/plugin-runner/custom-plugins" if IS_PRODUCTION else "./custom-plugins"
+
+# when we import plugins we'll use the module name directly so we need to add the plugin
+# directory to the path
+sys.path.append(PLUGIN_DIRECTORY)
+
+# a global dictionary of loaded plugins
+# TODO: create typings here for the subkeys
+LOADED_PLUGINS: dict = {}
+
+# a global dictionary of events to protocol class names
+EVENT_PROTOCOL_MAP: dict = {}
+
+
+class PluginRunner(PluginRunnerServicer):
+    """This process runs provided plugins that register interest in incoming events."""
+
+    def __init__(self) -> None:
+        self.statsd_client = statsd.StatsClient()
+        super().__init__()
+
+    sandbox: Sandbox
+
+    async def HandleEvent(self, request: Event, context: Any) -> EventResponse:
+        """This is invoked when an event comes in."""
+        event_start_time = time.time()
+        event_name = EventType.Name(request.type)
+        relevant_plugins = EVENT_PROTOCOL_MAP.get(event_name, [])
+
+        effect_list = []
+
+        for plugin_name in relevant_plugins:
+            plugin = LOADED_PLUGINS[plugin_name]
+            protocol_class = plugin["class"]
+            base_plugin_name = plugin_name.split(":")[0]
+
+            secrets = plugin.get("secrets", {})
+            secrets["graphql_jwt"] = token_for_plugin(plugin_name=plugin_name, audience="home")
+
+            try:
+                protocol = protocol_class(request, secrets)
+                classname = (
+                    protocol.__class__.__name__
+                    if isinstance(protocol, ClinicalQualityMeasure)
+                    else None
+                )
+
+                compute_start_time = time.time()
+                _effects = await asyncio.get_running_loop().run_in_executor(None, protocol.compute)
+                effects = [
+                    Effect(
+                        type=effect.type,
+                        payload=effect.payload,
+                        plugin_name=base_plugin_name,
+                        classname=classname,
+                    )
+                    for effect in _effects
+                ]
+                compute_duration = get_duration_ms(compute_start_time)
+
+                log.info(f"{plugin_name}.compute() completed ({compute_duration} ms)")
+                statsd_tags = tags_to_line_protocol({"plugin": plugin_name})
+                self.statsd_client.timing(
+                    f"plugins.protocol_duration_ms,{statsd_tags}",
+                    delta=compute_duration,
+                )
+            except Exception as e:
+                for error_line_with_newlines in traceback.format_exception(e):
+                    for error_line in error_line_with_newlines.split("\n"):
+                        log.error(error_line)
+                continue
+
+            effect_list += effects
+
+        event_duration = get_duration_ms(event_start_time)
+
+        # Don't log anything if a protocol didn't actually run.
+        if relevant_plugins:
+            log.info(f"Responded to Event {event_name} ({event_duration} ms)")
+            statsd_tags = tags_to_line_protocol({"event": event_name})
+            self.statsd_client.timing(
+                f"plugins.event_duration_ms,{statsd_tags}",
+                delta=event_duration,
+            )
+
+        yield EventResponse(success=True, effects=effect_list)
+
+    async def ReloadPlugins(
+        self, request: ReloadPluginsRequest, context: Any
+    ) -> ReloadPluginsResponse:
+        """This is invoked when we need to reload plugins."""
+        try:
+            load_plugins()
+            publish_message({"action": "restart"})
+        except ImportError:
+            yield ReloadPluginsResponse(success=False)
+        else:
+            yield ReloadPluginsResponse(success=True)
+
+
+def handle_hup_cb(_signum: int, _frame: Optional[FrameType]) -> None:
+    """handle_hup_cb."""
+    log.info("Received SIGHUP, reloading plugins...")
+    load_plugins()
+
+
+def sandbox_from_module_name(module_name: str) -> Any:
+    """Sandbox the code execution."""
+    spec = importlib.util.find_spec(module_name)
+
+    if not spec or not spec.origin:
+        raise Exception(f'Could not load plugin "{module_name}"')
+
+    origin = pathlib.Path(spec.origin)
+    source_code = origin.read_text()
+
+    sandbox = Sandbox(source_code)
+
+    return sandbox.execute()
+
+
+def load_or_reload_plugin(path: pathlib.Path) -> None:
+    """Given a path, load or reload a plugin."""
+    log.info(f"Loading {path}")
+
+    manifest_file = path / MANIFEST_FILE_NAME
+    manifest_json = manifest_file.read_text()
+
+    # the name is the folder name underneath the plugins directory
+    name = path.name
+
+    try:
+        manifest_json = json.loads(manifest_json)
+    except Exception as e:
+        log.error(f'Unable to load plugin "{name}": {e}')
+        return
+
+    secrets_file = path / SECRETS_FILE_NAME
+
+    secrets_json = {}
+    if secrets_file.exists():
+        try:
+            secrets_json = json.load(secrets_file.open())
+        except Exception as e:
+            log.error(f'Unable to load secrets for plugin "{name}": {str(e)}')
+
+    # TODO add existing schema validation from Michela here
+    try:
+        protocols = manifest_json["components"]["protocols"]
+    except Exception as e:
+        log.error(f'Unable to load plugin "{name}": {str(e)}')
+        return
+
+    for protocol in protocols:
+        # TODO add class colon validation to existing schema validation
+        # TODO when we encounter an exception here, disable the plugin in response
+        try:
+            protocol_module, protocol_class = protocol["class"].split(":")
+            name_and_class = f"{name}:{protocol_module}:{protocol_class}"
+        except ValueError:
+            log.error(f"Unable to parse class for plugin '{name}': '{protocol['class']}'")
+            continue
+
+        try:
+            if name_and_class in LOADED_PLUGINS:
+                log.info(f"Reloading plugin '{name_and_class}'")
+
+                result = sandbox_from_module_name(protocol_module)
+
+                LOADED_PLUGINS[name_and_class]["active"] = True
+
+                LOADED_PLUGINS[name_and_class]["class"] = result[protocol_class]
+                LOADED_PLUGINS[name_and_class]["sandbox"] = result
+                LOADED_PLUGINS[name_and_class]["secrets"] = secrets_json
+            else:
+                log.info(f"Loading plugin '{name_and_class}'")
+
+                result = sandbox_from_module_name(protocol_module)
+
+                LOADED_PLUGINS[name_and_class] = {
+                    "active": True,
+                    "class": result[protocol_class],
+                    "sandbox": result,
+                    "protocol": protocol,
+                    "secrets": secrets_json,
+                }
+        except Exception as err:
+            log.error(f"Error importing module '{name_and_class}': {err}")
+            for error_line in traceback.format_exception(err):
+                log.error(error_line)
+
+
+def refresh_event_type_map() -> None:
+    """Ensure the event subscriptions are up to date."""
+    global EVENT_PROTOCOL_MAP
+    EVENT_PROTOCOL_MAP = defaultdict(list)
+
+    for name, plugin in LOADED_PLUGINS.items():
+        if hasattr(plugin["class"], "RESPONDS_TO"):
+            responds_to = plugin["class"].RESPONDS_TO
+
+            if isinstance(responds_to, str):
+                EVENT_PROTOCOL_MAP[responds_to].append(name)
+            elif isinstance(responds_to, list):
+                for event in responds_to:
+                    EVENT_PROTOCOL_MAP[event].append(name)
+            else:
+                log.warning(f"Unknown RESPONDS_TO type: {type(responds_to)}")
+
+
+def load_plugins(specified_plugin_paths: list[str] | None = None) -> None:
+    """Load the plugins."""
+    # first mark each plugin as inactive since we want to remove it from
+    # LOADED_PLUGINS if it no longer exists on disk
+    for plugin in LOADED_PLUGINS.values():
+        plugin["active"] = False
+
+    if specified_plugin_paths is not None:
+        # convert to Paths
+        plugin_paths = [pathlib.Path(name) for name in specified_plugin_paths]
+
+        for plugin_path in plugin_paths:
+            # when we import plugins we'll use the module name directly so we need to add the plugin
+            # directory to the path
+            path_to_append = f"./{plugin_path.parent}"
+            sys.path.append(path_to_append)
+    else:
+        candidates = os.listdir(PLUGIN_DIRECTORY)
+
+        # convert to Paths
+        plugin_paths = [pathlib.Path(os.path.join(PLUGIN_DIRECTORY, name)) for name in candidates]
+
+    # get all directories under the plugin directory
+    plugin_paths = [path for path in plugin_paths if path.is_dir()]
+
+    # filter to only the directories containing a manifest file
+    plugin_paths = [path for path in plugin_paths if (path / MANIFEST_FILE_NAME).exists()]
+
+    # load or reload each plugin
+    for plugin_path in plugin_paths:
+        load_or_reload_plugin(plugin_path)
+
+    # if a plugin has been uninstalled/disabled remove it from LOADED_PLUGINS
+    for name, plugin in LOADED_PLUGINS.copy().items():
+        if not plugin["active"]:
+            del LOADED_PLUGINS[name]
+
+    refresh_event_type_map()
+
+
+_cleanup_coroutines = []
+
+
+async def serve(specified_plugin_paths: list[str] | None = None) -> None:
+    """Run the server."""
+    port = "50051"
+
+    server = grpc.aio.server()
+    server.add_insecure_port("127.0.0.1:" + port)
+
+    add_PluginRunnerServicer_to_server(PluginRunner(), server)
+
+    log.info(f"Starting server, listening on port {port}")
+
+    load_plugins(specified_plugin_paths)
+
+    await server.start()
+
+    async def server_graceful_shutdown() -> None:
+        log.info("Starting graceful shutdown...")
+        await server.stop(5)
+
+    _cleanup_coroutines.append(server_graceful_shutdown())
+
+    await server.wait_for_termination()
+
+
+def run_server(specified_plugin_paths: list[str] | None = None) -> None:
+    """Run the server."""
+    loop = asyncio.new_event_loop()
+
+    asyncio.set_event_loop(loop)
+
+    signal.signal(signal.SIGHUP, handle_hup_cb)
+
+    try:
+        loop.run_until_complete(serve(specified_plugin_paths))
+    except KeyboardInterrupt:
+        pass
+    finally:
+        loop.run_until_complete(*_cleanup_coroutines)
+        loop.close()
+
+
+if __name__ == "__main__":
+    run_server()

plugin_runner/plugin_synchronizer.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+
+import os
+import pickle
+from pathlib import Path
+from subprocess import STDOUT, CalledProcessError, check_output
+
+import redis
+
+APP_NAME = os.getenv("APP_NAME")
+
+CUSTOMER_IDENTIFIER = os.getenv("CUSTOMER_IDENTIFIER")
+PLUGINS_PUBSUB_CHANNEL = os.getenv("PLUGINS_PUBSUB_CHANNEL", default="plugins")
+
+CHANNEL_NAME = f"{CUSTOMER_IDENTIFIER}:{PLUGINS_PUBSUB_CHANNEL}"
+
+REDIS_ENDPOINT = os.getenv("REDIS_ENDPOINT", f"redis://{APP_NAME}-redis:6379")
+
+try:
+    CLIENT_ID = Path("/app/container-unique-id.txt").read_text()
+except FileNotFoundError:
+    CLIENT_ID = "non-unique"
+
+
+def get_client() -> tuple[redis.Redis, redis.client.PubSub]:
+    """Return a Redis client and pubsub object."""
+    client = redis.Redis.from_url(REDIS_ENDPOINT)
+    pubsub = client.pubsub()
+
+    return client, pubsub
+
+
+def publish_message(message: dict) -> None:
+    """Publish a message to the pubsub channel."""
+    client, _ = get_client()
+
+    message_with_id = {**message, "client_id": CLIENT_ID}
+
+    client.publish(CHANNEL_NAME, pickle.dumps(message_with_id))
+    client.close()
+
+
+def main() -> None:
+    """Listen for messages on the pubsub channel and restart the plugin-runner."""
+    print("plugin-synchronizer: starting")
+
+    _, pubsub = get_client()
+
+    pubsub.psubscribe(CHANNEL_NAME)
+
+    for message in pubsub.listen():
+        if not message:
+            continue
+
+        message_type = message.get("type", "")
+
+        if message_type != "pmessage":
+            continue
+
+        data = pickle.loads(message.get("data", pickle.dumps({})))
+
+        if "action" not in data or "client_id" not in data:
+            return
+
+        # Don't respond to our own messages
+        if data["client_id"] == CLIENT_ID:
+            return
+
+        if data["action"] == "restart":
+            # Run the plugin installer process
+            try:
+                print("plugin-synchronizer: installing plugins")
+                check_output(["./manage.py", "install_plugins_v2"], cwd="/app", stderr=STDOUT)
+            except CalledProcessError as e:
+                print("plugin-synchronizer: `./manage.py install_plugins_v2` failed:", e)
+
+            try:
+                print("plugin-synchronizer: sending SIGHUP to plugin-runner")
+                check_output(
+                    ["circusctl", "signal", "plugin-runner", "1"], cwd="/app", stderr=STDOUT
+                )
+            except CalledProcessError as e:
+                print("plugin-synchronizer: `circusctl signal plugin-runner 1` failed:", e)
+
+
+if __name__ == "__main__":
+    main()

plugin_runner/sandbox.py

@@ -0,0 +1,273 @@
+import ast
+import builtins
+from _ast import AnnAssign
+from functools import cached_property
+from typing import Any, cast
+
+from RestrictedPython import (
+    CompileResult,
+    PrintCollector,
+    RestrictingNodeTransformer,
+    compile_restricted_exec,
+    safe_builtins,
+    utility_builtins,
+)
+from RestrictedPython.Eval import default_guarded_getitem
+from RestrictedPython.Guards import (
+    guarded_iter_unpack_sequence,
+    guarded_unpack_sequence,
+)
+from RestrictedPython.transformer import (
+    ALLOWED_FUNC_NAMES,
+    FORBIDDEN_FUNC_NAMES,
+    copy_locations,
+)
+
+##
+# ALLOWED_MODULES
+#
+# The modules in this list are the only ones that can be imported in a sandboxed
+# runtime.
+#
+ALLOWED_MODULES = frozenset(
+    [
+        "_strptime",
+        "arrow",
+        "base64",
+        "cached_property",
+        "canvas_sdk.commands",
+        "canvas_sdk.data",
+        "canvas_sdk.effects",
+        "canvas_sdk.events",
+        "canvas_sdk.handlers",
+        "canvas_sdk.protocols",
+        "canvas_sdk.utils",
+        "canvas_sdk.v1",
+        "canvas_sdk.value_set",
+        "canvas_sdk.views",
+        "contextlib",
+        "datetime",
+        "dateutil",
+        "enum",
+        "functools",
+        "hashlib",
+        "hmac",
+        "http",
+        "json",
+        "logger",
+        "math",
+        "operator",
+        "pickletools",
+        "random",
+        "re",
+        "requests",
+        "string",
+        "time",
+        "traceback",
+        "typing",
+        "urllib",
+        "uuid",
+    ]
+)
+
+
+def _is_known_module(name: str) -> bool:
+    return any(name.startswith(m) for m in ALLOWED_MODULES)
+
+
+def _safe_import(name: str, *args: Any, **kwargs: Any) -> Any:
+    if not _is_known_module(name):
+        raise ImportError(f"{name!r} is not an allowed import.")
+    return __import__(name, *args, **kwargs)
+
+
+def _unrestricted(_ob: Any, *args: Any, **kwargs: Any) -> Any:
+    """Return the given object, unmodified."""
+    return _ob
+
+
+def _apply(_ob: Any, *args: Any, **kwargs: Any) -> Any:
+    """Call the bound method with args, support calling super().__init__()."""
+    return _ob(*args, **kwargs)
+
+
+class Sandbox:
+    """A restricted sandbox for safely executing arbitrary Python code."""
+
+    source_code: str
+    namespace: str
+
+    class Transformer(RestrictingNodeTransformer):
+        """A node transformer for customizing the sandbox compiler."""
+
+        def visit_AnnAssign(self, node: AnnAssign) -> AnnAssign:
+            """Allow type annotations."""
+            return node
+
+        def check_import_names(self, node: ast.ImportFrom) -> ast.AST:
+            """Check the names being imported.
+
+            This is a protection against rebinding dunder names like
+            _getitem_, _write_ via imports.
+
+            => 'from _a import x' is ok, because '_a' is not added to the scope.
+            """
+            for name in node.names:
+                if "*" in name.name and not _is_known_module(node.module):
+                    self.error(node, '"*" imports are not allowed.')
+                self.check_name(node, name.name)
+                if name.asname:
+                    self.check_name(node, name.asname)
+
+            return self.node_contents_visit(node)
+
+        def check_name(
+            self, node: ast.ImportFrom, name: str | None, allow_magic_methods: bool = False
+        ) -> None:
+            """Check names if they are allowed.
+
+            If ``allow_magic_methods is True`` names in `ALLOWED_FUNC_NAMES`
+            are additionally allowed although their names start with `_`.
+
+            Override to turn errors into warnings for leading underscores.
+            """
+            if name is None:
+                return
+
+            if (
+                name.startswith("_")
+                and name != "_"
+                and not (
+                    allow_magic_methods and name in ALLOWED_FUNC_NAMES and node.col_offset != 0
+                )
+            ):
+                self.warn(
+                    node,
+                    '"{name}" is an invalid variable name because it '
+                    'starts with "_"'.format(name=name),
+                )
+            elif name.endswith("__roles__"):
+                self.error(
+                    node,
+                    '"%s" is an invalid variable name because ' 'it ends with "__roles__".' % name,
+                )
+            elif name in FORBIDDEN_FUNC_NAMES:
+                self.error(node, f'"{name}" is a reserved name.')
+
+        def visit_Attribute(self, node: ast.Attribute) -> ast.AST:
+            """Checks and mutates attribute access/assignment.
+
+            'a.b' becomes '_getattr_(a, "b")'
+            'a.b = c' becomes '_write_(a).b = c'
+            'del a.b' becomes 'del _write_(a).b'
+
+            The _write_ function should return a security proxy.
+
+            Override to turn errors into warnings for leading underscores.
+            """
+            if node.attr.startswith("_") and node.attr != "_":
+                self.warn(
+                    node,
+                    '"{name}" is an invalid attribute name because it starts '
+                    'with "_".'.format(name=node.attr),
+                )
+
+            if node.attr.endswith("__roles__"):
+                self.error(
+                    node,
+                    '"{name}" is an invalid attribute name because it ends '
+                    'with "__roles__".'.format(name=node.attr),
+                )
+
+            if isinstance(node.ctx, ast.Load):
+                node = self.node_contents_visit(node)
+                new_node = ast.Call(
+                    func=ast.Name("_getattr_", ast.Load()),
+                    args=[node.value, ast.Str(node.attr)],
+                    keywords=[],
+                )
+
+                copy_locations(new_node, node)
+                return new_node
+
+            elif isinstance(node.ctx, (ast.Store, ast.Del)):
+                node = self.node_contents_visit(node)
+                new_value = ast.Call(
+                    func=ast.Name("_write_", ast.Load()), args=[node.value], keywords=[]
+                )
+
+                copy_locations(new_value, node.value)
+                node.value = new_value
+                return node
+
+            else:  # pragma: no cover
+                # Impossible Case only ctx Load, Store and Del are defined in ast.
+                raise NotImplementedError(f"Unknown ctx type: {type(node.ctx)}")
+
+    def __init__(self, source_code: str, namespace: str | None = None) -> None:
+        if source_code is None:
+            raise TypeError("source_code may not be None")
+        self.namespace = namespace or "protocols"
+        self.source_code = source_code
+
+    @cached_property
+    def scope(self) -> dict[str, Any]:
+        """Return the scope used for evaluation."""
+        return {
+            "__builtins__": {
+                **safe_builtins.copy(),
+                **utility_builtins.copy(),
+                "__import__": _safe_import,
+                "classmethod": builtins.classmethod,
+                "staticmethod": builtins.staticmethod,
+                "any": builtins.any,
+                "all": builtins.all,
+                "enumerate": builtins.enumerate,
+                "property": builtins.property,
+                "super": builtins.super,
+                "dict": builtins.dict,
+                "filter": builtins.filter,
+                "max": builtins.max,
+                "min": builtins.min,
+                "list": builtins.list,
+                "next": builtins.next,
+                "iter": builtins.iter,
+                "type": builtins.type,
+            },
+            "__metaclass__": type,
+            "__name__": self.namespace,
+            "_write_": _unrestricted,
+            "_getiter_": _unrestricted,
+            "_getitem_": default_guarded_getitem,
+            "_getattr_": getattr,
+            "_print_": PrintCollector,
+            "_apply_": _apply,
+            "_inplacevar_": _unrestricted,
+            "_iter_unpack_sequence_": guarded_iter_unpack_sequence,
+            "_unpack_sequence_": guarded_unpack_sequence,
+            "hasattr": hasattr,
+        }
+
+    @cached_property
+    def compile_result(self) -> CompileResult:
+        """Compile the source code into bytecode."""
+        return compile_restricted_exec(self.source_code, policy=self.Transformer)
+
+    @property
+    def errors(self) -> tuple[str, ...]:
+        """Return errors encountered when compiling the source code."""
+        return cast(tuple[str, ...], self.compile_result.errors)
+
+    @property
+    def warnings(self) -> tuple[str, ...]:
+        """Return warnings encountered when compiling the source code."""
+        return cast(tuple[str, ...], self.compile_result.warnings)
+
+    def execute(self) -> dict:
+        """Execute the given code in a restricted sandbox."""
+        if self.errors:
+            raise RuntimeError(f"Code is invalid: {self.errors}")
+
+        exec(self.compile_result.code, self.scope)
+
+        return self.scope