structlog-config 0.1.0__tar.gz → 0.4.0__tar.gz

structlog_config-0.4.0/PKG-INFO

@@ -0,0 +1,135 @@
+Metadata-Version: 2.3
+Name: structlog-config
+Version: 0.4.0
+Summary: A comprehensive structlog configuration with sensible defaults for development and production environments, featuring context management, exception formatting, and path prettification.
+Keywords: logging,structlog,json-logging,structured-logging
+Author: Michael Bianco
+Author-email: Michael Bianco <mike@mikebian.co>
+Requires-Dist: orjson>=3.10.15
+Requires-Dist: python-decouple-typed>=3.11.0
+Requires-Dist: python-ipware>=3.0.0
+Requires-Dist: structlog>=25.2.0
+Requires-Python: >=3.10
+Project-URL: Repository, https://github.com/iloveitaly/structlog-config
+Description-Content-Type: text/markdown
+
+# Opinionated Defaults for Structlog
+
+Logging is really important. Getting logging to work well in python feels like black magic: there's a ton of configuration
+across structlog, warnings, std loggers, fastapi + celery context, JSON logging in production, etc that requires lots of
+fiddling and testing to get working. I finally got this working for me in my [project template](https://github.com/iloveitaly/python-starter-template) and extracted this out into a nice package.
+
+Here are the main goals:
+
+* High performance JSON logging in production
+* All loggers, even plugin or system loggers, should route through the same formatter
+* Structured logging everywhere
+* Ability to easily set thread-local log context
+* Nice log formatters for stack traces, ORM ([ActiveModel/SQLModel](https://github.com/iloveitaly/activemodel)), etc
+* Ability to log level and output (i.e. file path) *by logger* for easy development debugging
+* If you are using fastapi, structured logging for access logs
+
+## Usage
+
+```python
+from structlog_config import configure_logger
+
+log = configure_logger()
+
+log.info("the log", key="value")
+```
+
+## TRACE Logging Level
+
+This package adds support for a custom `TRACE` logging level (level 5) that's even more verbose than `DEBUG`. This is useful for extremely detailed debugging scenarios.
+
+The `TRACE` level is automatically set up when you call `configure_logger()`. You can use it like any other logging level:
+
+```python
+import logging
+from structlog_config import configure_logger
+
+log = configure_logger()
+
+# Using structlog
+log.info("This is info")
+log.debug("This is debug") 
+log.trace("This is trace")  # Most verbose
+
+# Using stdlib logging
+logging.trace("Module-level trace message")
+logger = logging.getLogger(__name__)
+logger.trace("Instance trace message")
+```
+
+Set the log level to TRACE using the environment variable:
+
+```bash
+LOG_LEVEL=TRACE
+```
+
+## Stdlib Log Management
+
+By default, all stdlib loggers are:
+
+1. Given the same global logging level, with some default adjustments for noisy loggers (looking at you, `httpx`)
+2. Use a structlog formatter (you get structured logging, context, etc in any stdlib logger calls)
+3. The root processor is overwritten so any child loggers created after initialization will use the same formatter
+
+You can customize loggers by name (i.e. the name used in `logging.getLogger(__name__)`) using ENV variables.
+
+For example, if you wanted to [mimic `OPENAI_LOG` functionality](https://github.com/openai/openai-python/blob/de7c0e2d9375d042a42e3db6c17e5af9a5701a99/src/openai/_utils/_logs.py#L16):
+
+* `LOG_LEVEL_OPENAI=DEBUG`
+* `LOG_PATH_OPENAI=tmp/openai.log`
+* `LOG_LEVEL_HTTPX=DEBUG`
+* `LOG_PATH_HTTPX=tmp/openai.log`
+
+## FastAPI Access Logger
+
+Structured, simple access log with request timing to replace the default fastapi access log. Why?
+
+1. It's less verbose
+2. Uses structured logging params instead of string interpolation
+3. debug level logs any static assets
+
+Here's how to use it:
+
+1. [Disable fastapi's default logging.](https://github.com/iloveitaly/python-starter-template/blob/f54cb47d8d104987f2e4a668f9045a62e0d6818a/main.py#L55-L56)
+2. [Add the middleware to your FastAPI app.](https://github.com/iloveitaly/python-starter-template/blob/f54cb47d8d104987f2e4a668f9045a62e0d6818a/app/routes/middleware/__init__.py#L63-L65)
+
+## iPython
+
+Often it's helpful to update logging level within an iPython session. You can do this and make sure all loggers pick up on it.
+
+```
+%env LOG_LEVEL=DEBUG
+from structlog_config import configure_logger
+configure_logger()
+```
+
+## Related Projects
+
+* https://github.com/underyx/structlog-pretty
+* https://pypi.org/project/httpx-structlog/
+
+## References
+
+General logging:
+
+- https://github.com/replicate/cog/blob/2e57549e18e044982bd100e286a1929f50880383/python/cog/logging.py#L20
+- https://github.com/apache/airflow/blob/4280b83977cd5a53c2b24143f3c9a6a63e298acc/task_sdk/src/airflow/sdk/log.py#L187
+- https://github.com/kiwicom/structlog-sentry
+- https://github.com/jeremyh/datacube-explorer/blob/b289b0cde0973a38a9d50233fe0fff00e8eb2c8e/cubedash/logs.py#L40C21-L40C42
+- https://stackoverflow.com/questions/76256249/logging-in-the-open-ai-python-library/78214464#78214464
+- https://github.com/openai/openai-python/blob/de7c0e2d9375d042a42e3db6c17e5af9a5701a99/src/openai/_utils/_logs.py#L16
+- https://www.python-httpx.org/logging/
+
+FastAPI access logger:
+
+- https://github.com/iloveitaly/fastapi-logger/blob/main/fastapi_structlog/middleware/access_log.py#L70
+- https://github.com/fastapiutils/fastapi-utils/blob/master/fastapi_utils/timing.py
+- https://pypi.org/project/fastapi-structlog/
+- https://pypi.org/project/asgi-correlation-id/
+- https://gist.github.com/nymous/f138c7f06062b7c43c060bf03759c29e
+- https://github.com/sharu1204/fastapi-structlog/blob/master/app/main.py

structlog_config-0.4.0/README.md

@@ -0,0 +1,120 @@
+# Opinionated Defaults for Structlog
+
+Logging is really important. Getting logging to work well in python feels like black magic: there's a ton of configuration
+across structlog, warnings, std loggers, fastapi + celery context, JSON logging in production, etc that requires lots of
+fiddling and testing to get working. I finally got this working for me in my [project template](https://github.com/iloveitaly/python-starter-template) and extracted this out into a nice package.
+
+Here are the main goals:
+
+* High performance JSON logging in production
+* All loggers, even plugin or system loggers, should route through the same formatter
+* Structured logging everywhere
+* Ability to easily set thread-local log context
+* Nice log formatters for stack traces, ORM ([ActiveModel/SQLModel](https://github.com/iloveitaly/activemodel)), etc
+* Ability to log level and output (i.e. file path) *by logger* for easy development debugging
+* If you are using fastapi, structured logging for access logs
+
+## Usage
+
+```python
+from structlog_config import configure_logger
+
+log = configure_logger()
+
+log.info("the log", key="value")
+```
+
+## TRACE Logging Level
+
+This package adds support for a custom `TRACE` logging level (level 5) that's even more verbose than `DEBUG`. This is useful for extremely detailed debugging scenarios.
+
+The `TRACE` level is automatically set up when you call `configure_logger()`. You can use it like any other logging level:
+
+```python
+import logging
+from structlog_config import configure_logger
+
+log = configure_logger()
+
+# Using structlog
+log.info("This is info")
+log.debug("This is debug") 
+log.trace("This is trace")  # Most verbose
+
+# Using stdlib logging
+logging.trace("Module-level trace message")
+logger = logging.getLogger(__name__)
+logger.trace("Instance trace message")
+```
+
+Set the log level to TRACE using the environment variable:
+
+```bash
+LOG_LEVEL=TRACE
+```
+
+## Stdlib Log Management
+
+By default, all stdlib loggers are:
+
+1. Given the same global logging level, with some default adjustments for noisy loggers (looking at you, `httpx`)
+2. Use a structlog formatter (you get structured logging, context, etc in any stdlib logger calls)
+3. The root processor is overwritten so any child loggers created after initialization will use the same formatter
+
+You can customize loggers by name (i.e. the name used in `logging.getLogger(__name__)`) using ENV variables.
+
+For example, if you wanted to [mimic `OPENAI_LOG` functionality](https://github.com/openai/openai-python/blob/de7c0e2d9375d042a42e3db6c17e5af9a5701a99/src/openai/_utils/_logs.py#L16):
+
+* `LOG_LEVEL_OPENAI=DEBUG`
+* `LOG_PATH_OPENAI=tmp/openai.log`
+* `LOG_LEVEL_HTTPX=DEBUG`
+* `LOG_PATH_HTTPX=tmp/openai.log`
+
+## FastAPI Access Logger
+
+Structured, simple access log with request timing to replace the default fastapi access log. Why?
+
+1. It's less verbose
+2. Uses structured logging params instead of string interpolation
+3. debug level logs any static assets
+
+Here's how to use it:
+
+1. [Disable fastapi's default logging.](https://github.com/iloveitaly/python-starter-template/blob/f54cb47d8d104987f2e4a668f9045a62e0d6818a/main.py#L55-L56)
+2. [Add the middleware to your FastAPI app.](https://github.com/iloveitaly/python-starter-template/blob/f54cb47d8d104987f2e4a668f9045a62e0d6818a/app/routes/middleware/__init__.py#L63-L65)
+
+## iPython
+
+Often it's helpful to update logging level within an iPython session. You can do this and make sure all loggers pick up on it.
+
+```
+%env LOG_LEVEL=DEBUG
+from structlog_config import configure_logger
+configure_logger()
+```
+
+## Related Projects
+
+* https://github.com/underyx/structlog-pretty
+* https://pypi.org/project/httpx-structlog/
+
+## References
+
+General logging:
+
+- https://github.com/replicate/cog/blob/2e57549e18e044982bd100e286a1929f50880383/python/cog/logging.py#L20
+- https://github.com/apache/airflow/blob/4280b83977cd5a53c2b24143f3c9a6a63e298acc/task_sdk/src/airflow/sdk/log.py#L187
+- https://github.com/kiwicom/structlog-sentry
+- https://github.com/jeremyh/datacube-explorer/blob/b289b0cde0973a38a9d50233fe0fff00e8eb2c8e/cubedash/logs.py#L40C21-L40C42
+- https://stackoverflow.com/questions/76256249/logging-in-the-open-ai-python-library/78214464#78214464
+- https://github.com/openai/openai-python/blob/de7c0e2d9375d042a42e3db6c17e5af9a5701a99/src/openai/_utils/_logs.py#L16
+- https://www.python-httpx.org/logging/
+
+FastAPI access logger:
+
+- https://github.com/iloveitaly/fastapi-logger/blob/main/fastapi_structlog/middleware/access_log.py#L70
+- https://github.com/fastapiutils/fastapi-utils/blob/master/fastapi_utils/timing.py
+- https://pypi.org/project/fastapi-structlog/
+- https://pypi.org/project/asgi-correlation-id/
+- https://gist.github.com/nymous/f138c7f06062b7c43c060bf03759c29e
+- https://github.com/sharu1204/fastapi-structlog/blob/master/app/main.py

{structlog_config-0.1.0 → structlog_config-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "structlog-config"
-version = "0.1.0"
+version = "0.4.0"
 description = "A comprehensive structlog configuration with sensible defaults for development and production environments, featuring context management, exception formatting, and path prettification."
 keywords = ["logging", "structlog", "json-logging", "structured-logging"]
 readme = "README.md"
@@ -8,14 +8,19 @@ requires-python = ">=3.10"
 dependencies = [
     "orjson>=3.10.15",
     "python-decouple-typed>=3.11.0",
+    "python-ipware>=3.0.0",
     "structlog>=25.2.0",
 ]
 authors = [{ name = "Michael Bianco", email = "mike@mikebian.co" }]
 urls = { "Repository" = "https://github.com/iloveitaly/structlog-config" }
 
 [build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
+requires = ["uv_build>=0.8.11,<0.9.0"]
+build-backend = "uv_build"
+
+[tool.uv.build-backend]
+# avoids the src/ directory structure
+module-root = ""
 
 [dependency-groups]
 debugging-extras = [

{structlog_config-0.1.0 → structlog_config-0.4.0}/structlog_config/__init__.py

@@ -1,9 +1,10 @@
-import logging
-from typing import Protocol
+from contextlib import _GeneratorContextManager
+from typing import Generator, Protocol
 
 import orjson
 import structlog
 import structlog.dev
+from decouple import config
 from structlog.processors import ExceptionRenderer
 from structlog.tracebacks import ExceptionDictTransformer
 from structlog.typing import FilteringBoundLogger
@@ -16,17 +17,19 @@ from structlog_config.formatters import (
     simplify_activemodel_objects,
 )
 
-from . import packages
-from .constants import NO_COLOR, PYTHON_LOG_PATH
+from . import (
+    packages,
+    trace,  # noqa: F401
+)
+from .constants import NO_COLOR, package_logger
 from .environments import is_production, is_pytest, is_staging
+from .levels import get_environment_log_level_as_string
 from .stdlib_logging import (
-    get_environment_log_level_as_string,
     redirect_stdlib_loggers,
 )
+from .trace import setup_trace
 from .warnings import redirect_showwarnings
 
-package_logger = logging.getLogger(__name__)
-
 
 def log_processors_for_mode(json_logger: bool) -> list[structlog.types.Processor]:
     if json_logger:
@@ -99,11 +102,19 @@ def _logger_factory(json_logger: bool):
     In production, optimized for speed (https://www.structlog.org/en/stable/performance.html)
     """
 
+    # avoid a constant for this ENV so we can mutate within tests
+    python_log_path = config("PYTHON_LOG_PATH", default=None)
+
     if json_logger:
+        # TODO I guess we could support this, but the assumption is stdout is going to be used in prod environments
+        if python_log_path:
+            package_logger.warning(
+                "PYTHON_LOG_PATH is not supported with a JSON logger, forcing stdout"
+            )
         return structlog.BytesLoggerFactory()
 
-    if PYTHON_LOG_PATH:
-        python_log = open(PYTHON_LOG_PATH, "a", encoding="utf-8")
+    if python_log_path:
+        python_log = open(python_log_path, "a", encoding="utf-8")
         return structlog.PrintLoggerFactory(file=python_log)
 
     # Default case
@@ -118,7 +129,7 @@ class LoggerWithContext(FilteringBoundLogger, Protocol):
     want to replicate.
     """
 
-    def context(self, *args, **kwargs) -> None:
+    def context(self, *args, **kwargs) -> _GeneratorContextManager[None, None, None]:
         "context manager to temporarily set and clear logging context"
         ...
 
@@ -158,6 +169,8 @@ def configure_logger(
         json_logger: Optional flag to use JSON logging. If None, defaults to
             production or staging environment sourced from PYTHON_ENV.
     """
+    setup_trace()
+
     # Reset structlog configuration to make sure we're starting fresh
     # This is important for tests where configure_logger might be called multiple times
     structlog.reset_defaults()

structlog_config-0.4.0/structlog_config/constants.py

@@ -0,0 +1,16 @@
+import logging
+import os
+
+from decouple import config
+
+PYTHONASYNCIODEBUG = config("PYTHONASYNCIODEBUG", default=False, cast=bool)
+"this is a builtin env var, we check for it to ensure we don't silence this log level"
+
+NO_COLOR = "NO_COLOR" in os.environ
+"support NO_COLOR standard https://no-color.org"
+
+package_logger = logging.getLogger(__name__)
+"strange name to not be confused with all of the log-related names floating around"
+
+TRACE_LOG_LEVEL = 5
+"Custom log level for trace logging, lower than DEBUG"

{structlog_config-0.1.0 → structlog_config-0.4.0}/structlog_config/env_config.py

@@ -10,7 +10,7 @@ LOG_LEVEL_PATTERN = re.compile(r"^LOG_LEVEL_(.+)$")
 LOG_PATH_PATTERN = re.compile(r"^LOG_PATH_(.+)$")
 
 
-def get_custom_logger_configs() -> dict[str, dict[str, str]]:
+def get_custom_logger_config() -> dict[str, dict[str, str]]:
     """
     Parse environment variables to extract custom logger configurations.
 

{structlog_config-0.1.0 → structlog_config-0.4.0}/structlog_config/fastapi_access_logger.py

@@ -3,13 +3,16 @@ from urllib.parse import quote
 
 import structlog
 from fastapi import FastAPI
+from python_ipware import IpWare
 from starlette.middleware.base import RequestResponseEndpoint
 from starlette.requests import Request
 from starlette.responses import Response
 from starlette.routing import Match, Mount
 from starlette.types import Scope
+from starlette.websockets import WebSocket
 
 log = structlog.get_logger("access_log")
+ipw = IpWare()
 
 
 def get_route_name(app: FastAPI, scope: Scope, prefix: str = "") -> str:
@@ -63,6 +66,42 @@ def get_client_addr(scope: Scope) -> str:
     return f"{ip}:{port}"
 
 
+def client_ip_from_request(request: Request | WebSocket) -> str | None:
+    """
+    Get the client IP address from the request.
+
+    Headers are not case-sensitive.
+
+    Uses ipware library to properly extract client IP from various proxy headers.
+    Fallback to direct client connection if no proxy headers found.
+    """
+    headers = request.headers
+
+    # TODO this seems really inefficient, we should just rewrite the ipware into this repo :/
+    # Convert Starlette headers to format expected by ipware (HTTP_ prefixed)
+    # ipware expects headers in WSGI/Django-style meta format where HTTP headers
+    # are prefixed with "HTTP_" and dashes become underscores.
+    # See: https://github.com/un33k/python-ipware/blob/main/python_ipware/python_ipware.py#L33-L40
+    meta_dict = {}
+    for name, value in headers.items():
+        # Convert header name to HTTP_ prefixed format
+        meta_key = f"HTTP_{name.upper().replace('-', '_')}"
+        meta_dict[meta_key] = value
+
+    # Use ipware to extract IP from headers
+    ip, trusted_route = ipw.get_client_ip(meta=meta_dict)
+    if ip:
+        log.debug(
+            "extracted client IP from headers", ip=ip, trusted_route=trusted_route
+        )
+        return str(ip)
+
+    # Fallback to direct client connection
+    host = request.client.host if request.client else None
+
+    return host
+
+
 # TODO we should look at the static asset logic and pull the prefix path from tha
 def is_static_assets_request(scope: Scope) -> bool:
     """Check if the request is for static assets. Pretty naive check.
@@ -73,13 +112,32 @@ def is_static_assets_request(scope: Scope) -> bool:
     Returns:
         bool: True if the request is for static assets, False otherwise.
     """
-    return scope["path"].endswith(".css") or scope["path"].endswith(".js")
+    return (
+        scope["path"].endswith(".css")
+        or scope["path"].endswith(".js")
+        # .map files are attempted when devtools are enabled
+        or scope["path"].endswith(".js.map")
+        or scope["path"].endswith(".ico")
+        or scope["path"].endswith(".png")
+        or scope["path"].endswith(".jpg")
+        or scope["path"].endswith(".jpeg")
+        or scope["path"].endswith(".gif")
+    )
 
 
 def add_middleware(
     app: FastAPI,
 ) -> None:
-    """Use this method to add this middleware to your fastapi application."""
+    """
+    Add better access logging to fastapi:
+
+    >>> from structlog_config import fastapi_access_logger
+    >>> fastapi_access_logger.add_middleware(app)
+
+    You'll also want to disable the default uvicorn logs:
+
+    >>> uvicorn.run(..., log_config=None, access_log=False)
+    """
 
     @app.middleware("http")
     async def access_log_middleware(
@@ -108,7 +166,7 @@ def add_middleware(
             method=scope["method"],
             path=scope["path"],
             query=scope["query_string"].decode(),
-            client_ip=get_client_addr(scope),
+            client_ip=client_ip_from_request(request),
             route=route_name,
         )
 

{structlog_config-0.1.0 → structlog_config-0.4.0}/structlog_config/formatters.py

@@ -77,6 +77,7 @@ def pretty_traceback_exception_formatter(sio: TextIO, exc_info: ExcInfo) -> None
     from pretty_traceback.formatting import exc_to_traceback_str
 
     _, exc_value, traceback = exc_info
+    # TODO support local_stack_only env var support
     formatted_exception = exc_to_traceback_str(exc_value, traceback, color=not NO_COLOR)
     sio.write("\n" + formatted_exception)
 

structlog_config-0.4.0/structlog_config/levels.py

@@ -0,0 +1,35 @@
+import logging
+
+from decouple import config
+
+
+def get_environment_log_level_as_string() -> str:
+    level = config("LOG_LEVEL", default="INFO", cast=str).upper()
+
+    if not level.strip():
+        level = "INFO"
+
+    return level
+
+
+def compare_log_levels(left: str, right: str) -> int:
+    """
+    Compare log levels using logging.getLevelNamesMapping for accurate int values.
+
+    Example:
+    >>> compare_log_levels("DEBUG", "INFO")
+    -1  # DEBUG is less than INFO
+
+    Asks the question "Is INFO higher than DEBUG?"
+    """
+    level_map = logging.getLevelNamesMapping()
+    left_level = level_map.get(left, left)
+    right_level = level_map.get(right, right)
+
+    # TODO should more gracefully fail here, but let's see what happens
+    if not isinstance(left_level, int) or not isinstance(right_level, int):
+        raise ValueError(
+            f"Invalid log level comparison: {left} ({type(left_level)}) vs {right} ({type(right_level)})"
+        )
+
+    return left_level - right_level