flagsmith-common 3.4.0__tar.gz → 3.5.0__tar.gz

{flagsmith_common-3.4.0 → flagsmith_common-3.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flagsmith-common
-Version: 3.4.0
+Version: 3.5.0
 Summary: Flagsmith's common library
 Author: Matthew Elwell, Gagan Trivedi, Kim Gustyr, Zach Aysan, Francesco Lo Franco, Rodrigo López Dato, Evandro Myller, Wadii Zaim
 License-Expression: BSD-3-Clause
@@ -20,7 +20,10 @@ Requires-Dist: gunicorn>=19.1 ; extra == 'common-core'
 Requires-Dist: prometheus-client>=0.0.16 ; extra == 'common-core'
 Requires-Dist: psycopg2-binary>=2.9,<3 ; extra == 'common-core'
 Requires-Dist: requests ; extra == 'common-core'
+Requires-Dist: sentry-sdk>=2.0.0,<3.0.0 ; extra == 'common-core'
 Requires-Dist: simplejson>=3,<4 ; extra == 'common-core'
+Requires-Dist: structlog>=24.4,<26 ; extra == 'common-core'
+Requires-Dist: typing-extensions ; extra == 'common-core'
 Requires-Dist: simplejson ; extra == 'flagsmith-schemas'
 Requires-Dist: typing-extensions ; extra == 'flagsmith-schemas'
 Requires-Dist: flagsmith-flag-engine>6 ; extra == 'flagsmith-schemas'

{flagsmith_common-3.4.0 → flagsmith_common-3.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "flagsmith-common"
-version = "3.4.0"
+version = "3.5.0"
 description = "Flagsmith's common library"
 requires-python = ">=3.11,<4.0"
 dependencies = []
@@ -19,7 +19,10 @@ optional-dependencies = { test-tools = [
     "prometheus-client (>=0.0.16)",
     "psycopg2-binary (>=2.9,<3)",
     "requests",
+    "sentry-sdk (>=2.0.0,<3.0.0)",
     "simplejson (>=3,<4)",
+    "structlog (>=24.4,<26)",
+    "typing_extensions",
 ], task-processor = [
     "backoff (>=2.2.1,<3.0.0)",
     "django (>4,<6)",

{flagsmith_common-3.4.0 → flagsmith_common-3.5.0}/src/common/core/constants.py

@@ -1 +1,3 @@
 DEFAULT_PROMETHEUS_MULTIPROC_DIR_NAME = "flagsmith-prometheus"
+
+LOGGING_DEFAULT_ROOT_LOG_LEVEL = "WARNING"

flagsmith_common-3.5.0/src/common/core/logging.py

@@ -0,0 +1,181 @@
+import json
+import logging
+import logging.config
+import os
+import sys
+import threading
+from typing import Any
+
+import structlog
+import structlog.contextvars
+import structlog.dev
+import structlog.processors
+import structlog.stdlib
+from structlog.typing import EventDict, Processor, WrappedLogger
+from typing_extensions import TypedDict
+
+from common.core.constants import LOGGING_DEFAULT_ROOT_LOG_LEVEL
+from common.core.sentry import sentry_processor
+
+logger = logging.getLogger(__name__)
+
+
+class JsonRecord(TypedDict, extra_items=Any, total=False):  # type: ignore[call-arg]  # TODO https://github.com/python/mypy/issues/18176
+    levelname: str
+    message: str
+    timestamp: str
+    logger_name: str
+    pid: int | None
+    thread_name: str | None
+    exc_info: str
+
+
+def setup_logging(
+    log_level: str = "INFO",
+    log_format: str = "generic",
+    logging_configuration_file: str | None = None,
+    application_loggers: list[str] | None = None,
+    extra_foreign_processors: list[Processor] | None = None,
+) -> None:
+    """
+    Set up logging for the application.
+
+    This should be called early, before Django settings are loaded, to ensure
+    that all log output is properly formatted from the start.
+
+    Args:
+        log_level: The log level for application code (e.g. "DEBUG", "INFO").
+        log_format: Either "generic" or "json".
+        logging_configuration_file: Path to a JSON logging config file.
+            If provided, this takes precedence over other format options.
+        application_loggers: Top-level logger names for application packages.
+            These loggers are set to ``log_level`` while the root logger uses
+            ``max(log_level, WARNING)`` to suppress noise from third-party
+            libraries. If ``log_level`` is DEBUG, everything logs at DEBUG.
+        extra_foreign_processors: Additional structlog processors to run in
+            the ``foreign_pre_chain`` for stdlib log records (e.g. the
+            Gunicorn access log field extractor).
+    """
+    if logging_configuration_file:
+        with open(logging_configuration_file) as f:
+            config = json.load(f)
+        logging.config.dictConfig(config)
+    else:
+        log_level_int = logging.getLevelNamesMapping()[log_level.upper()]
+        root_level_int = logging.getLevelNamesMapping()[LOGGING_DEFAULT_ROOT_LOG_LEVEL]
+        # Suppress third-party noise at WARNING, but if the user requests
+        # DEBUG, honour that for the entire process.
+        effective_root_level = (
+            log_level_int if log_level_int < logging.INFO else root_level_int
+        )
+
+        dict_config: dict[str, Any] = {
+            "version": 1,
+            "disable_existing_loggers": False,
+            "handlers": {
+                "console": {
+                    "class": "logging.StreamHandler",
+                    "stream": "ext://sys.stdout",
+                    "level": log_level,
+                },
+            },
+            "root": {
+                "level": effective_root_level,
+                "handlers": ["console"],
+            },
+            "loggers": {
+                name: {"level": log_level, "handlers": [], "propagate": True}
+                for name in application_loggers or []
+            },
+        }
+        logging.config.dictConfig(dict_config)
+
+    setup_structlog(
+        log_format=log_format, extra_foreign_processors=extra_foreign_processors
+    )
+
+
+def map_event_to_json_record(
+    logger: WrappedLogger,
+    method_name: str,
+    event_dict: EventDict,
+) -> EventDict:
+    """Map structlog fields to match :class:`JsonRecord` output schema."""
+    # Remove foreign record args injected by pass_foreign_args so they
+    # don't leak into the rendered JSON output.
+    event_dict.pop("positional_args", None)
+    record: JsonRecord = {
+        "message": event_dict.pop("event", ""),
+        "levelname": event_dict.pop("level", "").upper(),
+        "logger_name": event_dict.pop("logger", ""),
+        "pid": os.getpid(),
+        "thread_name": threading.current_thread().name,
+    }
+    if "exception" in event_dict:
+        record["exc_info"] = event_dict.pop("exception")
+    # Merge remaining structlog keys (e.g. extra_key from bind()) with the
+    # canonical record so they appear in the JSON output.
+    event_dict.update(record)
+    return event_dict
+
+
+def setup_structlog(
+    log_format: str,
+    extra_foreign_processors: list[Processor] | None = None,
+) -> None:
+    """Configure structlog to route through stdlib logging."""
+
+    if log_format == "json":
+        renderer_processors: list[Processor] = [
+            map_event_to_json_record,
+            structlog.processors.JSONRenderer(),
+        ]
+    else:
+        colors = sys.stdout.isatty() and structlog.dev._has_colors
+        renderer_processors = [
+            structlog.dev.ConsoleRenderer(colors=colors),
+        ]
+
+    foreign_pre_chain: list[Processor] = [
+        structlog.contextvars.merge_contextvars,
+        structlog.stdlib.add_logger_name,
+        structlog.stdlib.add_log_level,
+        structlog.processors.TimeStamper(fmt="iso"),
+        structlog.processors.format_exc_info,
+        structlog.stdlib.ExtraAdder(),
+        *(extra_foreign_processors or []),
+    ]
+
+    formatter = structlog.stdlib.ProcessorFormatter(
+        processors=[
+            structlog.stdlib.ProcessorFormatter.remove_processors_meta,
+            *renderer_processors,
+        ],
+        foreign_pre_chain=foreign_pre_chain,
+        pass_foreign_args=True,
+    )
+
+    # Replace the formatter on existing root handlers with ProcessorFormatter.
+    root = logging.getLogger()
+    for handler in root.handlers:
+        handler.setFormatter(formatter)
+
+    structlog.configure(
+        processors=[
+            structlog.contextvars.merge_contextvars,
+            structlog.stdlib.filter_by_level,
+            structlog.stdlib.add_logger_name,
+            structlog.stdlib.add_log_level,
+            structlog.stdlib.PositionalArgumentsFormatter(),
+            structlog.processors.StackInfoRenderer(),
+            structlog.processors.UnicodeDecoder(),
+            structlog.processors.format_exc_info,
+            structlog.processors.TimeStamper(fmt="iso"),
+            sentry_processor,
+            structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
+        ],
+        wrapper_class=structlog.stdlib.BoundLogger,
+        context_class=dict,
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        cache_logger_on_first_use=True,
+    )

{flagsmith_common-3.4.0 → flagsmith_common-3.5.0}/src/common/core/main.py

@@ -8,8 +8,13 @@ from tempfile import mkdtemp
 from django.core.management import (
     execute_from_command_line as django_execute_from_command_line,
 )
+from environs import Env
 
 from common.core.cli import healthcheck
+from common.core.logging import setup_logging
+from common.gunicorn.processors import make_gunicorn_access_processor
+
+env = Env()
 
 logger = logging.getLogger(__name__)
 
@@ -32,7 +37,18 @@ def ensure_cli_env() -> typing.Generator[None, None, None]:
     """
     ctx = contextlib.ExitStack()
 
-    # TODO @khvn26 Move logging setup to here
+    # Set up logging early, before Django settings are loaded.
+    setup_logging(
+        log_level=env.str("LOG_LEVEL", "INFO"),
+        log_format=env.str("LOG_FORMAT", "generic"),
+        logging_configuration_file=env.str("LOGGING_CONFIGURATION_FILE", None),
+        application_loggers=env.list("APPLICATION_LOGGERS", []) or None,
+        extra_foreign_processors=[
+            make_gunicorn_access_processor(
+                env.list("ACCESS_LOG_EXTRA_ITEMS", []) or None,
+            ),
+        ],
+    )
 
     # Prometheus multiproc support
     if not os.environ.get("PROMETHEUS_MULTIPROC_DIR"):
@@ -67,12 +83,13 @@ def execute_from_command_line(argv: list[str]) -> None:
             "checktaskprocessorthreadhealth": healthcheck.main,
         }[subcommand]
     except (IndexError, KeyError):
-        django_execute_from_command_line(argv)
+        logger.info("Invoking Django")
     else:
-        subcommand_main(
+        return subcommand_main(
             argv[2:],
             prog=f"{os.path.basename(argv[0])} {subcommand}",
         )
+    django_execute_from_command_line(argv)
 
 
 def main(argv: list[str] = sys.argv) -> None:

flagsmith_common-3.5.0/src/common/core/sentry.py

@@ -0,0 +1,22 @@
+import sentry_sdk
+from structlog.typing import EventDict, WrappedLogger
+
+_SKIP_CONTEXT_FIELDS = frozenset({"event", "level", "timestamp", "_record"})
+
+
+def sentry_processor(
+    logger: WrappedLogger,
+    method_name: str,
+    event_dict: EventDict,
+) -> EventDict:
+    """
+    Structlog processor that enriches Sentry with structured context and tags.
+
+    Since structlog routes through stdlib, Sentry's LoggingIntegration
+    automatically captures ERROR+ logs as Sentry events. This processor
+    adds structured context on top of that.
+    """
+    context = {k: v for k, v in event_dict.items() if k not in _SKIP_CONTEXT_FIELDS}
+    sentry_sdk.set_context("structlog", context)
+
+    return event_dict

flagsmith_common-3.5.0/src/common/gunicorn/logging.py

@@ -0,0 +1,66 @@
+import logging
+import os
+from datetime import timedelta
+from typing import Any
+
+from gunicorn.config import Config  # type: ignore[import-untyped]
+from gunicorn.http.message import Request  # type: ignore[import-untyped]
+from gunicorn.http.wsgi import Response  # type: ignore[import-untyped]
+from gunicorn.instrument.statsd import (  # type: ignore[import-untyped]
+    Statsd as StatsdGunicornLogger,
+)
+
+from common.gunicorn import metrics
+from common.gunicorn.utils import get_extra
+
+
+class PrometheusGunicornLogger(StatsdGunicornLogger):  # type: ignore[misc]
+    """Gunicorn logger that records Prometheus metrics on each access log entry."""
+
+    def access(
+        self,
+        resp: Response,
+        req: Request,
+        environ: dict[str, Any],
+        request_time: timedelta,
+    ) -> None:
+        super().access(resp, req, environ, request_time)
+        duration_seconds = (
+            request_time.seconds + float(request_time.microseconds) / 10**6
+        )
+        labels = {
+            # To avoid cardinality explosion, we use a resolved Django route
+            # instead of raw path.
+            # The Django route is set by `RouteLoggerMiddleware`.
+            "route": get_extra(environ=environ, key="route") or "",
+            "method": environ.get("REQUEST_METHOD") or "",
+            "response_status": resp.status_code,
+        }
+        metrics.flagsmith_http_server_request_duration_seconds.labels(**labels).observe(
+            duration_seconds
+        )
+        metrics.flagsmith_http_server_requests_total.labels(**labels).inc()
+        metrics.flagsmith_http_server_response_size_bytes.labels(**labels).observe(
+            getattr(resp, "sent", 0),
+        )
+
+
+class GunicornJsonCapableLogger(PrometheusGunicornLogger):
+    """Gunicorn logger that integrates with the application logging setup."""
+
+    def setup(self, cfg: Config) -> None:
+        super().setup(cfg)
+
+        # Error log always propagates to root.
+        for handler in self.error_log.handlers[:]:
+            self.error_log.removeHandler(handler)
+        self.error_log.propagate = True
+
+        # In JSON mode, replace the access log formatter with the root's
+        # ProcessorFormatter. In generic mode, keep Gunicorn's CLF formatter.
+        # The handler itself is preserved so ACCESS_LOG_LOCATION is respected.
+        root_handlers = logging.getLogger().handlers
+        if os.environ.get("LOG_FORMAT") == "json" and root_handlers:
+            root_formatter = root_handlers[0].formatter
+            for handler in self.access_log.handlers:
+                handler.setFormatter(root_formatter)

{flagsmith_common-3.4.0 → flagsmith_common-3.5.0}/src/common/gunicorn/metrics.py

@@ -1,5 +1,4 @@
 import prometheus_client
-from django.conf import settings
 
 from common.gunicorn.constants import HTTP_SERVER_RESPONSE_SIZE_DEFAULT_BUCKETS
 from common.prometheus import Histogram
@@ -18,9 +17,5 @@ flagsmith_http_server_response_size_bytes = Histogram(
     "flagsmith_http_server_response_size_bytes",
     "HTTP response size in bytes.",
     ["route", "method", "response_status"],
-    buckets=getattr(
-        settings,
-        "PROMETHEUS_HTTP_SERVER_RESPONSE_SIZE_HISTOGRAM_BUCKETS",
-        HTTP_SERVER_RESPONSE_SIZE_DEFAULT_BUCKETS,
-    ),
+    buckets=HTTP_SERVER_RESPONSE_SIZE_DEFAULT_BUCKETS,
 )

flagsmith_common-3.5.0/src/common/gunicorn/processors.py

@@ -0,0 +1,74 @@
+from datetime import datetime
+
+from structlog.typing import EventDict, Processor, WrappedLogger
+
+from common.gunicorn.constants import (
+    WSGI_EXTRA_SUFFIX_TO_CATEGORY,
+    wsgi_extra_key_regex,
+)
+
+
+def make_gunicorn_access_processor(
+    access_log_extra_items: list[str] | None = None,
+) -> Processor:
+    """Create a processor that extracts structured fields from Gunicorn access logs.
+
+    Gunicorn populates ``record.args`` with a dict of request/response data
+    (keyed by format variables like ``h``, ``m``, ``s``, ``U``, etc.).  This
+    processor detects those records and promotes the data into the event dict
+    so it flows through the normal rendering pipeline.
+
+    Pass the returned processor to :func:`~common.core.logging.setup_logging`
+    via ``extra_foreign_processors``.
+    """
+
+    def processor(
+        logger: WrappedLogger,
+        method_name: str,
+        event_dict: EventDict,
+    ) -> EventDict:
+        record = event_dict.get("_record")
+        if record is None or record.name != "gunicorn.access":
+            return event_dict
+        # Gunicorn passes request data as a dict in record.args.
+        # By the time foreign_pre_chain runs, ProcessorFormatter has
+        # cleared args on its copy; positional_args has the originals.
+        args = event_dict.get("positional_args", record.args)
+        if not isinstance(args, dict):
+            return event_dict
+
+        url = args.get("U", "")
+        if q := args.get("q"):
+            url += f"?{q}"
+
+        if t := args.get("t"):
+            event_dict["time"] = datetime.strptime(
+                t, "[%d/%b/%Y:%H:%M:%S %z]"
+            ).isoformat()
+        event_dict["path"] = url
+        event_dict["remote_ip"] = args.get("h", "")
+        event_dict["method"] = args.get("m", "")
+        event_dict["status"] = str(args.get("s", ""))
+        event_dict["user_agent"] = args.get("a", "")
+        event_dict["duration_in_ms"] = args.get("M", 0)
+        event_dict["response_size_in_bytes"] = args.get("B") or 0
+
+        if access_log_extra_items:
+            for extra_key in access_log_extra_items:
+                extra_key_lower = extra_key.lower()
+                if (
+                    (extra_value := args.get(extra_key_lower))
+                    and (re_match := wsgi_extra_key_regex.match(extra_key_lower))
+                    and (
+                        category := WSGI_EXTRA_SUFFIX_TO_CATEGORY.get(
+                            re_match.group("suffix")
+                        )
+                    )
+                ):
+                    event_dict.setdefault(category, {})[re_match.group("key")] = (
+                        extra_value
+                    )
+
+        return event_dict
+
+    return processor

{flagsmith_common-3.4.0 → flagsmith_common-3.5.0}/src/common/prometheus/utils.py

@@ -1,13 +1,10 @@
 import importlib
 
 import prometheus_client
-from django.conf import settings
 from prometheus_client.metrics import MetricWrapperBase
 from prometheus_client.multiprocess import MultiProcessCollector
 
-
-class Histogram(prometheus_client.Histogram):
-    DEFAULT_BUCKETS = settings.PROMETHEUS_HISTOGRAM_BUCKETS
+Histogram = prometheus_client.Histogram
 
 
 def get_registry() -> prometheus_client.CollectorRegistry:

flagsmith_common-3.4.0/src/common/core/logging.py

@@ -1,24 +0,0 @@
-import json
-import logging
-from typing import Any
-
-
-class JsonFormatter(logging.Formatter):
-    """Custom formatter for json logs."""
-
-    def get_json_record(self, record: logging.LogRecord) -> dict[str, Any]:
-        formatted_message = record.getMessage()
-        json_record = {
-            "levelname": record.levelname,
-            "message": formatted_message,
-            "timestamp": self.formatTime(record, self.datefmt),
-            "logger_name": record.name,
-            "pid": record.process,
-            "thread_name": record.threadName,
-        }
-        if record.exc_info:
-            json_record["exc_info"] = self.formatException(record.exc_info)
-        return json_record
-
-    def format(self, record: logging.LogRecord) -> str:
-        return json.dumps(self.get_json_record(record))

flagsmith_common-3.4.0/src/common/gunicorn/logging.py

@@ -1,120 +0,0 @@
-import logging
-import sys
-from datetime import datetime, timedelta
-from typing import TYPE_CHECKING, Any
-
-from django.conf import settings
-from gunicorn.config import Config  # type: ignore[import-untyped]
-from gunicorn.http.message import Request  # type: ignore[import-untyped]
-from gunicorn.http.wsgi import Response  # type: ignore[import-untyped]
-from gunicorn.instrument.statsd import (  # type: ignore[import-untyped]
-    Statsd as StatsdGunicornLogger,
-)
-
-from common.core.logging import JsonFormatter
-from common.gunicorn import metrics
-from common.gunicorn.constants import (
-    WSGI_EXTRA_PREFIX,
-    WSGI_EXTRA_SUFFIX_TO_CATEGORY,
-    wsgi_extra_key_regex,
-)
-from common.gunicorn.utils import get_extra
-
-
-class GunicornAccessLogJsonFormatter(JsonFormatter):
-    def _get_extra(self, record_args: dict[str, Any]) -> dict[str, Any]:
-        ret: dict[str, dict[str, Any]] = {}
-
-        extra_items_to_log: list[str] | None
-        if extra_items_to_log := getattr(settings, "ACCESS_LOG_EXTRA_ITEMS", None):
-            # We expect the extra items to be in the form of
-            # Gunicorn's access log format string for
-            # request headers, response headers and environ variables
-            # without the % prefix, e.g. "{origin}i" or "{flagsmith.environment_id}e"
-            # https://docs.gunicorn.org/en/stable/settings.html#access-log-format
-            for extra_key in extra_items_to_log:
-                extra_key_lower = extra_key.lower()
-                if (
-                    (extra_value := record_args.get(extra_key_lower))
-                    and (re_match := wsgi_extra_key_regex.match(extra_key_lower))
-                    and (
-                        extra_category := WSGI_EXTRA_SUFFIX_TO_CATEGORY.get(
-                            re_match.group("suffix")
-                        )
-                    )
-                ):
-                    ret.setdefault(extra_category, {})[re_match.group("key")] = (
-                        extra_value
-                    )
-
-        return ret
-
-    def get_json_record(self, record: logging.LogRecord) -> dict[str, Any]:
-        args = record.args
-
-        if TYPE_CHECKING:
-            assert isinstance(args, dict)
-
-        url = args["U"]
-        if q := args["q"]:
-            url += f"?{q}"
-
-        return {
-            **super().get_json_record(record),
-            "time": datetime.strptime(args["t"], "[%d/%b/%Y:%H:%M:%S %z]").isoformat(),
-            "path": url,
-            "remote_ip": args["h"],
-            "route": args.get(f"{{{WSGI_EXTRA_PREFIX}route}}e") or "",
-            "method": args["m"],
-            "status": str(args["s"]),
-            "user_agent": args["a"],
-            "duration_in_ms": args["M"],
-            "response_size_in_bytes": args["B"] or 0,
-            **self._get_extra(args),
-        }
-
-
-class PrometheusGunicornLogger(StatsdGunicornLogger):  # type: ignore[misc]
-    def access(
-        self,
-        resp: Response,
-        req: Request,
-        environ: dict[str, Any],
-        request_time: timedelta,
-    ) -> None:
-        super().access(resp, req, environ, request_time)
-        duration_seconds = (
-            request_time.seconds + float(request_time.microseconds) / 10**6
-        )
-        labels = {
-            # To avoid cardinality explosion, we use a resolved Django route
-            # instead of raw path.
-            # The Django route is set by `RouteLoggerMiddleware`.
-            "route": get_extra(environ=environ, key="route") or "",
-            "method": environ.get("REQUEST_METHOD") or "",
-            "response_status": resp.status_code,
-        }
-        metrics.flagsmith_http_server_request_duration_seconds.labels(**labels).observe(
-            duration_seconds
-        )
-        metrics.flagsmith_http_server_requests_total.labels(**labels).inc()
-        metrics.flagsmith_http_server_response_size_bytes.labels(**labels).observe(
-            getattr(resp, "sent", 0),
-        )
-
-
-class GunicornJsonCapableLogger(PrometheusGunicornLogger):
-    def setup(self, cfg: Config) -> None:
-        super().setup(cfg)
-        if getattr(settings, "LOG_FORMAT", None) == "json":
-            self._set_handler(
-                self.error_log,
-                cfg.errorlog,
-                JsonFormatter(),
-            )
-            self._set_handler(
-                self.access_log,
-                cfg.accesslog,
-                GunicornAccessLogJsonFormatter(),
-                stream=sys.stdout,
-            )