canvas 0.63.0__py3-none-any.whl → 0.89.0__py3-none-any.whl

logger/logger.py

@@ -2,26 +2,11 @@ import logging
 import os
 from typing import Any
 
-import redis
+from django.conf import settings
+from logstash_async.handler import AsynchronousLogstashHandler
 
-from pubsub.pubsub import Publisher
-
-
-class PubSubLogHandler(logging.Handler):
-    """Custom logging handler that publishes logs to a pub/sub channel."""
-
-    def __init__(self) -> None:
-        self.publisher = Publisher()
-        logging.Handler.__init__(self=self)
-
-    def emit(self, record: Any) -> None:
-        """Publishes the log message to the pub/sub channel."""
-        message = self.format(record)
-
-        try:
-            self.publisher.publish(message)
-        except redis.ConnectionError as e:
-            print(f"PubSubLogHandler: failed to log message due to redis error: {e}")
+from logger.logstash import LogstashFormatterECS
+from logger.pubsub import PubSubLogHandler
 
 
 class PluginLogger:
@@ -31,13 +16,13 @@ class PluginLogger:
         self.logger = logging.getLogger("plugin_runner_logger")
         self.logger.setLevel(logging.INFO)
 
-        log_prefix = os.getenv("HOSTNAME", "")
+        log_prefix = f"{os.getenv('HOSTNAME', '?')}: {os.getenv('APTIBLE_PROCESS_INDEX', '?')}"
 
         if log_prefix != "":
             log_prefix = f"[{log_prefix}] "
 
         formatter = logging.Formatter(
-            f"plugin_runner {log_prefix}%(levelname)s %(asctime)s %(message)s"
+            f"plugin-runner {log_prefix}%(levelname)s %(asctime)s %(message)s"
         )
 
         streaming_handler = logging.StreamHandler()
@@ -45,29 +30,43 @@ class PluginLogger:
 
         self.logger.addHandler(streaming_handler)
 
-        if os.getenv("REDIS_ENDPOINT"):
+        if settings.REDIS_ENDPOINT:
             pubsub_handler = PubSubLogHandler()
             pubsub_handler.setFormatter(formatter)
 
             self.logger.addHandler(pubsub_handler)
 
-    def debug(self, message: Any) -> None:
+        if settings.LOGSTASH_HOST:
+            logstash_handler = AsynchronousLogstashHandler(
+                host=settings.LOGSTASH_HOST,
+                port=settings.LOGSTASH_PORT,
+                database_path=None,
+                transport=settings.LOGSTASH_PROTOCOL,
+            )
+            logstash_handler.setFormatter(LogstashFormatterECS())
+            self.logger.addHandler(logstash_handler)
+
+    def debug(self, message: Any, *args: Any, **kwargs: Any) -> None:
         """Logs a debug message."""
-        self.logger.debug(message)
+        self.logger.debug(message, *args, **kwargs)
 
-    def info(self, message: Any) -> None:
+    def info(self, message: Any, *args: Any, **kwargs: Any) -> None:
         """Logs an info message."""
-        self.logger.info(message)
+        self.logger.info(message, *args, **kwargs)
 
-    def warning(self, message: Any) -> None:
+    def warning(self, message: Any, *args: Any, **kwargs: Any) -> None:
         """Logs a warning message."""
-        self.logger.warning(message)
+        self.logger.warning(message, *args, **kwargs)
 
-    def error(self, message: Any) -> None:
+    def error(self, message: Any, *args: Any, **kwargs: Any) -> None:
         """Logs an error message."""
-        self.logger.error(message)
+        self.logger.error(message, *args, **kwargs)
+
+    def exception(self, message: Any, *args: Any, **kwargs: Any) -> None:
+        """Convenience method for logging an ERROR with exception information."""
+        self.logger.exception(message, *args, **kwargs)
 
-    def critical(self, message: Any) -> None:
+    def critical(self, message: Any, *args: Any, **kwargs: Any) -> None:
         """Logs a critical message."""
         self.logger.critical(message)
 

logger/logstash.py

@@ -0,0 +1,282 @@
+import contextlib
+import datetime
+import json
+import logging
+import re
+import sys
+import traceback
+from logging import LogRecord
+from types import TracebackType
+from typing import Any
+
+import requests
+from django.conf import settings
+
+
+class HttpTransport:
+    """
+    Send messages to Logstash in V1 format.
+    """
+
+    def __init__(self, host: str, **kwargs: Any) -> None:
+        self.url = host
+
+        self.session = requests.Session()
+        self.session.headers.update({"Content-Type": "application/json"})
+
+    def send(self, events: list[Any], **kwargs: Any) -> None:
+        """Send events to Logstash."""
+        for event in events:
+            try:
+                self.session.post(self.url, data=event)
+            except (KeyboardInterrupt, SystemExit):
+                raise
+            except BaseException as e:
+                print("Logstash exception", e)
+
+    def close(self) -> None:
+        """Close the transport."""
+        pass
+
+
+ExcInfo = tuple[type[BaseException], BaseException, TracebackType | None]
+
+
+def _figure_out_exc_info(v: Any) -> ExcInfo:
+    """
+    Depending on the Python version will try to do the smartest thing possible
+    to transform *v* into an ``exc_info`` tuple.
+    """
+    if isinstance(v, BaseException):
+        return v.__class__, v, v.__traceback__
+
+    if isinstance(v, tuple):
+        return v
+
+    if v:
+        return sys.exc_info()  # type: ignore[return-value]
+
+    return v
+
+
+def _json_default(obj: Any) -> str:
+    """
+    Coerce everything to strings. All objects representing time get output as
+    ISO8601.
+    """
+    if isinstance(obj, datetime.datetime | datetime.date | datetime.time):
+        return obj.isoformat()
+
+    return str(obj)
+
+
+class LogstashFormatterV1(logging.Formatter):
+    """
+    A custom formatter to prepare logs to be shipped out to logstash V1 format.
+    """
+
+    unwanted_fields = [
+        "args",
+        "created",
+        "filename",
+        "funcName",
+        "levelno",
+        "lineno",
+        "module",
+        "msecs",
+        "name",
+        "pathname",
+        "process",
+        "processName",
+        "relativeCreated",
+        "request",
+        "response",
+        "threadName",
+    ]
+
+    mappings = {
+        "levelname": "log_level",
+        "name": "log_name",
+    }
+
+    def __init__(self) -> None:
+        super().__init__()
+
+        self.defaults = {
+            "app": "plugin-runner",
+            "customer": settings.CUSTOMER_IDENTIFIER,
+            "plugin_context": True,
+        }
+
+        self.source_host = settings.HOSTNAME
+
+    @staticmethod
+    def get_location(fields: dict) -> str:
+        """
+        Format the log location in a way that saves some bytes.
+        """
+        path = "?"
+
+        if "pathname" in fields:
+            path = re.sub(r"^/plugin-runner/", "", fields["pathname"])
+            path = re.sub(r"^/usr/local/lib/python3.\d\d/dist-packages/", "", path)
+            path = re.sub(r"^/venvs/plugin-runner/lib/python3.\d\d/site-packages/", "", path)
+        elif "filename" in fields:
+            path = fields["filename"]
+
+        return "{}:{}".format(path, fields.get("lineno", "?"))
+
+    @staticmethod
+    def get_method(fields: dict) -> str:
+        """
+        Get the method that originated the log.
+        """
+        return "{}.{}:{}".format(
+            fields.get("name", "?"), fields.get("module", "?"), fields.get("funcName", "?")
+        )
+
+    def format(self, record: LogRecord) -> str:
+        """
+        Format a log record to JSON, if the message is a dict assume an empty
+        message and use the dict as additional fields.
+        """
+        fields = record.__dict__.copy()
+
+        if "msg" in fields and isinstance(fields["msg"], dict):
+            msg = fields.pop("msg")
+
+            # if the dict has an "event" key, use that as the message
+            # to avoid conflicts with other message types that use event as an object.
+            if "event" in msg:
+                fields["message"] = msg.pop("event")
+
+            fields.update(msg)
+        elif "msg" in fields and "message" not in fields:
+            msg = record.getMessage()
+
+            del fields["msg"]
+
+            with contextlib.suppress(KeyError, IndexError, ValueError):
+                msg = msg.format(**fields)
+
+            fields["message"] = msg
+
+        if "exc_info" in fields:
+            if fields["exc_info"]:
+                exc_info = _figure_out_exc_info(fields["exc_info"])
+                fields["exception_message"] = str(exc_info[1])
+                fields["exception_type"] = f"{exc_info[0].__module__}.{exc_info[0].__qualname__}"
+                fields["stack_trace"] = traceback.format_exception(*exc_info)
+            del fields["exc_info"]
+
+        if "exc_text" in fields and not fields["exc_text"]:
+            del fields["exc_text"]
+
+        now = datetime.datetime.now(datetime.UTC)
+
+        base_log = {
+            "@timestamp": now.isoformat(timespec="milliseconds").replace("+00:00", "Z"),
+            "@version": 1,
+            "location": self.get_location(fields),
+            "method": self.get_method(fields),
+            "source_host": self.source_host,
+        }
+
+        for field in self.unwanted_fields:
+            if field in fields:
+                del fields[field]
+
+        # Remove None fields (like `params` most of the time) and empty
+        # argument lists (like `args` most of the time)
+        fields = {key: value for key, value in fields.items() if value is not None and value != []}
+
+        # Rename fields with mapping set
+        for old_name, new_name in self.mappings.items():
+            if old_name in fields:
+                fields[new_name] = fields.pop(old_name)
+
+        base_log.update(fields)
+
+        log_record = self.defaults.copy()
+        log_record.update(base_log)
+
+        return json.dumps(log_record, default=_json_default)
+
+
+class LogstashFormatterECS(logging.Formatter):
+    """
+    A custom formatter to prepare logs to be shipped out to logstash ECS format.
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+
+        self.defaults = {
+            "service": {"name": "plugin-runner"},
+            "labels": {"customer": settings.CUSTOMER_IDENTIFIER},
+            "host": {"name": settings.HOSTNAME},
+        }
+
+    def format(self, record: LogRecord) -> str:
+        """
+        Format a log record to JSON, if the message is a dict assume an empty
+        message and use the dict as additional fields.
+        """
+        fields = record.__dict__.copy()
+
+        if "msg" in fields and isinstance(fields["msg"], dict):
+            msg = fields.pop("msg")
+
+            # if the dict has an "event" key, use that as the message
+            # to avoid conflicts with other message types that use event as an object.
+            if "event" in msg:
+                fields["message"] = msg.pop("event")
+
+            fields.update(msg)
+        elif "msg" in fields and "message" not in fields:
+            msg = record.getMessage()
+
+            del fields["msg"]
+
+            with contextlib.suppress(KeyError, IndexError, ValueError):
+                msg = msg.format(**fields)
+
+            fields["message"] = msg
+
+        if "exc_info" in fields:
+            if fields["exc_info"]:
+                exc_info = _figure_out_exc_info(fields["exc_info"])
+                fields["exception_message"] = str(exc_info[1])
+                fields["exception_type"] = f"{exc_info[0].__module__}.{exc_info[0].__qualname__}"
+                fields["stack_trace"] = traceback.format_exception(*exc_info)
+            del fields["exc_info"]
+
+        if "exc_text" in fields and not fields["exc_text"]:
+            del fields["exc_text"]
+
+        now = datetime.datetime.now(datetime.UTC)
+
+        log_record = {
+            **self.defaults,
+            "@timestamp": now.isoformat(timespec="milliseconds").replace("+00:00", "Z"),
+            "message": fields.get("message", ""),
+            "log": {
+                "level": fields.get("levelname", ""),
+            },
+            **(
+                {
+                    "error": {
+                        "message": fields["exception_message"],
+                        "type": fields["exception_type"],
+                        "stack_trace": fields["stack_trace"],
+                    }
+                }
+                if "exception_message" in fields
+                else {}
+            ),
+        }
+
+        return json.dumps(log_record, default=_json_default)
+
+
+__exports__ = ()

logger/pubsub.py

@@ -0,0 +1,26 @@
+import logging
+from typing import Any
+
+import redis
+
+from pubsub.pubsub import Publisher
+
+
+class PubSubLogHandler(logging.Handler):
+    """Custom logging handler that publishes logs to a pub/sub channel."""
+
+    def __init__(self) -> None:
+        self.publisher = Publisher()
+        logging.Handler.__init__(self=self)
+
+    def emit(self, record: Any) -> None:
+        """Publishes the log message to the pub/sub channel."""
+        message = self.format(record)
+
+        try:
+            self.publisher.publish(message)
+        except redis.ConnectionError as e:
+            print(f"PubSubLogHandler: failed to log message due to redis error: {e}")
+
+
+__exports__ = ()