structlog-config 0.1.0__py3-none-any.whl → 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- structlog_config/__init__.py +23 -10
- structlog_config/constants.py +8 -1
- structlog_config/env_config.py +1 -1
- structlog_config/fastapi_access_logger.py +61 -3
- structlog_config/formatters.py +1 -0
- structlog_config/levels.py +35 -0
- structlog_config/stdlib_logging.py +70 -39
- structlog_config/trace.py +62 -0
- structlog_config/warnings.py +6 -16
- structlog_config-0.4.0.dist-info/METADATA +135 -0
- structlog_config-0.4.0.dist-info/RECORD +14 -0
- structlog_config-0.4.0.dist-info/WHEEL +4 -0
- structlog_config-0.1.0.dist-info/METADATA +0 -62
- structlog_config-0.1.0.dist-info/RECORD +0 -12
- structlog_config-0.1.0.dist-info/WHEEL +0 -4
structlog_config/__init__.py
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import
|
|
2
|
-
from typing import Protocol
|
|
1
|
+
from contextlib import _GeneratorContextManager
|
|
2
|
+
from typing import Generator, Protocol
|
|
3
3
|
|
|
4
4
|
import orjson
|
|
5
5
|
import structlog
|
|
6
6
|
import structlog.dev
|
|
7
|
+
from decouple import config
|
|
7
8
|
from structlog.processors import ExceptionRenderer
|
|
8
9
|
from structlog.tracebacks import ExceptionDictTransformer
|
|
9
10
|
from structlog.typing import FilteringBoundLogger
|
|
@@ -16,17 +17,19 @@ from structlog_config.formatters import (
|
|
|
16
17
|
simplify_activemodel_objects,
|
|
17
18
|
)
|
|
18
19
|
|
|
19
|
-
from . import
|
|
20
|
-
|
|
20
|
+
from . import (
|
|
21
|
+
packages,
|
|
22
|
+
trace, # noqa: F401
|
|
23
|
+
)
|
|
24
|
+
from .constants import NO_COLOR, package_logger
|
|
21
25
|
from .environments import is_production, is_pytest, is_staging
|
|
26
|
+
from .levels import get_environment_log_level_as_string
|
|
22
27
|
from .stdlib_logging import (
|
|
23
|
-
get_environment_log_level_as_string,
|
|
24
28
|
redirect_stdlib_loggers,
|
|
25
29
|
)
|
|
30
|
+
from .trace import setup_trace
|
|
26
31
|
from .warnings import redirect_showwarnings
|
|
27
32
|
|
|
28
|
-
package_logger = logging.getLogger(__name__)
|
|
29
|
-
|
|
30
33
|
|
|
31
34
|
def log_processors_for_mode(json_logger: bool) -> list[structlog.types.Processor]:
|
|
32
35
|
if json_logger:
|
|
@@ -99,11 +102,19 @@ def _logger_factory(json_logger: bool):
|
|
|
99
102
|
In production, optimized for speed (https://www.structlog.org/en/stable/performance.html)
|
|
100
103
|
"""
|
|
101
104
|
|
|
105
|
+
# avoid a constant for this ENV so we can mutate within tests
|
|
106
|
+
python_log_path = config("PYTHON_LOG_PATH", default=None)
|
|
107
|
+
|
|
102
108
|
if json_logger:
|
|
109
|
+
# TODO I guess we could support this, but the assumption is stdout is going to be used in prod environments
|
|
110
|
+
if python_log_path:
|
|
111
|
+
package_logger.warning(
|
|
112
|
+
"PYTHON_LOG_PATH is not supported with a JSON logger, forcing stdout"
|
|
113
|
+
)
|
|
103
114
|
return structlog.BytesLoggerFactory()
|
|
104
115
|
|
|
105
|
-
if
|
|
106
|
-
python_log = open(
|
|
116
|
+
if python_log_path:
|
|
117
|
+
python_log = open(python_log_path, "a", encoding="utf-8")
|
|
107
118
|
return structlog.PrintLoggerFactory(file=python_log)
|
|
108
119
|
|
|
109
120
|
# Default case
|
|
@@ -118,7 +129,7 @@ class LoggerWithContext(FilteringBoundLogger, Protocol):
|
|
|
118
129
|
want to replicate.
|
|
119
130
|
"""
|
|
120
131
|
|
|
121
|
-
def context(self, *args, **kwargs) -> None:
|
|
132
|
+
def context(self, *args, **kwargs) -> _GeneratorContextManager[None, None, None]:
|
|
122
133
|
"context manager to temporarily set and clear logging context"
|
|
123
134
|
...
|
|
124
135
|
|
|
@@ -158,6 +169,8 @@ def configure_logger(
|
|
|
158
169
|
json_logger: Optional flag to use JSON logging. If None, defaults to
|
|
159
170
|
production or staging environment sourced from PYTHON_ENV.
|
|
160
171
|
"""
|
|
172
|
+
setup_trace()
|
|
173
|
+
|
|
161
174
|
# Reset structlog configuration to make sure we're starting fresh
|
|
162
175
|
# This is important for tests where configure_logger might be called multiple times
|
|
163
176
|
structlog.reset_defaults()
|
structlog_config/constants.py
CHANGED
|
@@ -1,9 +1,16 @@
|
|
|
1
|
+
import logging
|
|
1
2
|
import os
|
|
2
3
|
|
|
3
4
|
from decouple import config
|
|
4
5
|
|
|
5
|
-
PYTHON_LOG_PATH = config("PYTHON_LOG_PATH", default=None)
|
|
6
6
|
PYTHONASYNCIODEBUG = config("PYTHONASYNCIODEBUG", default=False, cast=bool)
|
|
7
|
+
"this is a builtin env var, we check for it to ensure we don't silence this log level"
|
|
7
8
|
|
|
8
9
|
NO_COLOR = "NO_COLOR" in os.environ
|
|
9
10
|
"support NO_COLOR standard https://no-color.org"
|
|
11
|
+
|
|
12
|
+
package_logger = logging.getLogger(__name__)
|
|
13
|
+
"strange name to not be confused with all of the log-related names floating around"
|
|
14
|
+
|
|
15
|
+
TRACE_LOG_LEVEL = 5
|
|
16
|
+
"Custom log level for trace logging, lower than DEBUG"
|
structlog_config/env_config.py
CHANGED
|
@@ -10,7 +10,7 @@ LOG_LEVEL_PATTERN = re.compile(r"^LOG_LEVEL_(.+)$")
|
|
|
10
10
|
LOG_PATH_PATTERN = re.compile(r"^LOG_PATH_(.+)$")
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
def
|
|
13
|
+
def get_custom_logger_config() -> dict[str, dict[str, str]]:
|
|
14
14
|
"""
|
|
15
15
|
Parse environment variables to extract custom logger configurations.
|
|
16
16
|
|
|
@@ -3,13 +3,16 @@ from urllib.parse import quote
|
|
|
3
3
|
|
|
4
4
|
import structlog
|
|
5
5
|
from fastapi import FastAPI
|
|
6
|
+
from python_ipware import IpWare
|
|
6
7
|
from starlette.middleware.base import RequestResponseEndpoint
|
|
7
8
|
from starlette.requests import Request
|
|
8
9
|
from starlette.responses import Response
|
|
9
10
|
from starlette.routing import Match, Mount
|
|
10
11
|
from starlette.types import Scope
|
|
12
|
+
from starlette.websockets import WebSocket
|
|
11
13
|
|
|
12
14
|
log = structlog.get_logger("access_log")
|
|
15
|
+
ipw = IpWare()
|
|
13
16
|
|
|
14
17
|
|
|
15
18
|
def get_route_name(app: FastAPI, scope: Scope, prefix: str = "") -> str:
|
|
@@ -63,6 +66,42 @@ def get_client_addr(scope: Scope) -> str:
|
|
|
63
66
|
return f"{ip}:{port}"
|
|
64
67
|
|
|
65
68
|
|
|
69
|
+
def client_ip_from_request(request: Request | WebSocket) -> str | None:
|
|
70
|
+
"""
|
|
71
|
+
Get the client IP address from the request.
|
|
72
|
+
|
|
73
|
+
Headers are not case-sensitive.
|
|
74
|
+
|
|
75
|
+
Uses ipware library to properly extract client IP from various proxy headers.
|
|
76
|
+
Fallback to direct client connection if no proxy headers found.
|
|
77
|
+
"""
|
|
78
|
+
headers = request.headers
|
|
79
|
+
|
|
80
|
+
# TODO this seems really inefficient, we should just rewrite the ipware into this repo :/
|
|
81
|
+
# Convert Starlette headers to format expected by ipware (HTTP_ prefixed)
|
|
82
|
+
# ipware expects headers in WSGI/Django-style meta format where HTTP headers
|
|
83
|
+
# are prefixed with "HTTP_" and dashes become underscores.
|
|
84
|
+
# See: https://github.com/un33k/python-ipware/blob/main/python_ipware/python_ipware.py#L33-L40
|
|
85
|
+
meta_dict = {}
|
|
86
|
+
for name, value in headers.items():
|
|
87
|
+
# Convert header name to HTTP_ prefixed format
|
|
88
|
+
meta_key = f"HTTP_{name.upper().replace('-', '_')}"
|
|
89
|
+
meta_dict[meta_key] = value
|
|
90
|
+
|
|
91
|
+
# Use ipware to extract IP from headers
|
|
92
|
+
ip, trusted_route = ipw.get_client_ip(meta=meta_dict)
|
|
93
|
+
if ip:
|
|
94
|
+
log.debug(
|
|
95
|
+
"extracted client IP from headers", ip=ip, trusted_route=trusted_route
|
|
96
|
+
)
|
|
97
|
+
return str(ip)
|
|
98
|
+
|
|
99
|
+
# Fallback to direct client connection
|
|
100
|
+
host = request.client.host if request.client else None
|
|
101
|
+
|
|
102
|
+
return host
|
|
103
|
+
|
|
104
|
+
|
|
66
105
|
# TODO we should look at the static asset logic and pull the prefix path from tha
|
|
67
106
|
def is_static_assets_request(scope: Scope) -> bool:
|
|
68
107
|
"""Check if the request is for static assets. Pretty naive check.
|
|
@@ -73,13 +112,32 @@ def is_static_assets_request(scope: Scope) -> bool:
|
|
|
73
112
|
Returns:
|
|
74
113
|
bool: True if the request is for static assets, False otherwise.
|
|
75
114
|
"""
|
|
76
|
-
return
|
|
115
|
+
return (
|
|
116
|
+
scope["path"].endswith(".css")
|
|
117
|
+
or scope["path"].endswith(".js")
|
|
118
|
+
# .map files are attempted when devtools are enabled
|
|
119
|
+
or scope["path"].endswith(".js.map")
|
|
120
|
+
or scope["path"].endswith(".ico")
|
|
121
|
+
or scope["path"].endswith(".png")
|
|
122
|
+
or scope["path"].endswith(".jpg")
|
|
123
|
+
or scope["path"].endswith(".jpeg")
|
|
124
|
+
or scope["path"].endswith(".gif")
|
|
125
|
+
)
|
|
77
126
|
|
|
78
127
|
|
|
79
128
|
def add_middleware(
|
|
80
129
|
app: FastAPI,
|
|
81
130
|
) -> None:
|
|
82
|
-
"""
|
|
131
|
+
"""
|
|
132
|
+
Add better access logging to fastapi:
|
|
133
|
+
|
|
134
|
+
>>> from structlog_config import fastapi_access_logger
|
|
135
|
+
>>> fastapi_access_logger.add_middleware(app)
|
|
136
|
+
|
|
137
|
+
You'll also want to disable the default uvicorn logs:
|
|
138
|
+
|
|
139
|
+
>>> uvicorn.run(..., log_config=None, access_log=False)
|
|
140
|
+
"""
|
|
83
141
|
|
|
84
142
|
@app.middleware("http")
|
|
85
143
|
async def access_log_middleware(
|
|
@@ -108,7 +166,7 @@ def add_middleware(
|
|
|
108
166
|
method=scope["method"],
|
|
109
167
|
path=scope["path"],
|
|
110
168
|
query=scope["query_string"].decode(),
|
|
111
|
-
client_ip=
|
|
169
|
+
client_ip=client_ip_from_request(request),
|
|
112
170
|
route=route_name,
|
|
113
171
|
)
|
|
114
172
|
|
structlog_config/formatters.py
CHANGED
|
@@ -77,6 +77,7 @@ def pretty_traceback_exception_formatter(sio: TextIO, exc_info: ExcInfo) -> None
|
|
|
77
77
|
from pretty_traceback.formatting import exc_to_traceback_str
|
|
78
78
|
|
|
79
79
|
_, exc_value, traceback = exc_info
|
|
80
|
+
# TODO support local_stack_only env var support
|
|
80
81
|
formatted_exception = exc_to_traceback_str(exc_value, traceback, color=not NO_COLOR)
|
|
81
82
|
sio.write("\n" + formatted_exception)
|
|
82
83
|
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
from decouple import config
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def get_environment_log_level_as_string() -> str:
|
|
7
|
+
level = config("LOG_LEVEL", default="INFO", cast=str).upper()
|
|
8
|
+
|
|
9
|
+
if not level.strip():
|
|
10
|
+
level = "INFO"
|
|
11
|
+
|
|
12
|
+
return level
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def compare_log_levels(left: str, right: str) -> int:
|
|
16
|
+
"""
|
|
17
|
+
Compare log levels using logging.getLevelNamesMapping for accurate int values.
|
|
18
|
+
|
|
19
|
+
Example:
|
|
20
|
+
>>> compare_log_levels("DEBUG", "INFO")
|
|
21
|
+
-1 # DEBUG is less than INFO
|
|
22
|
+
|
|
23
|
+
Asks the question "Is INFO higher than DEBUG?"
|
|
24
|
+
"""
|
|
25
|
+
level_map = logging.getLevelNamesMapping()
|
|
26
|
+
left_level = level_map.get(left, left)
|
|
27
|
+
right_level = level_map.get(right, right)
|
|
28
|
+
|
|
29
|
+
# TODO should more gracefully fail here, but let's see what happens
|
|
30
|
+
if not isinstance(left_level, int) or not isinstance(right_level, int):
|
|
31
|
+
raise ValueError(
|
|
32
|
+
f"Invalid log level comparison: {left} ({type(left_level)}) vs {right} ({type(right_level)})"
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
return left_level - right_level
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Redirect all stdlib loggers to use the structlog configuration.
|
|
3
|
+
"""
|
|
4
|
+
|
|
1
5
|
import logging
|
|
2
6
|
import sys
|
|
3
7
|
from pathlib import Path
|
|
@@ -5,52 +9,51 @@ from pathlib import Path
|
|
|
5
9
|
import structlog
|
|
6
10
|
from decouple import config
|
|
7
11
|
|
|
8
|
-
from structlog_config.env_config import get_custom_logger_configs
|
|
9
|
-
|
|
10
12
|
from .constants import PYTHONASYNCIODEBUG
|
|
13
|
+
from .env_config import get_custom_logger_config
|
|
11
14
|
from .environments import is_production, is_staging
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
15
|
+
from .levels import (
|
|
16
|
+
compare_log_levels,
|
|
17
|
+
get_environment_log_level_as_string,
|
|
18
|
+
)
|
|
16
19
|
|
|
17
20
|
|
|
18
21
|
def reset_stdlib_logger(
|
|
19
|
-
logger_name: str,
|
|
20
|
-
default_structlog_handler: logging.Handler,
|
|
21
|
-
level_override: str | None = None,
|
|
22
|
+
logger_name: str, default_structlog_handler: logging.Handler, level_override: str
|
|
22
23
|
):
|
|
23
24
|
std_logger = logging.getLogger(logger_name)
|
|
24
25
|
std_logger.propagate = False
|
|
25
26
|
std_logger.handlers = []
|
|
26
27
|
std_logger.addHandler(default_structlog_handler)
|
|
27
|
-
|
|
28
|
-
if level_override:
|
|
29
|
-
std_logger.setLevel(level_override)
|
|
28
|
+
std_logger.setLevel(level_override)
|
|
30
29
|
|
|
31
30
|
|
|
32
31
|
def redirect_stdlib_loggers(json_logger: bool):
|
|
33
32
|
"""
|
|
34
33
|
Redirect all standard logging module loggers to use the structlog configuration.
|
|
35
34
|
|
|
35
|
+
- json_loggers determines if logs are rendered as JSON or not
|
|
36
|
+
- The stdlib log stream is used to write logs to the output device (normally, stdout)
|
|
37
|
+
|
|
36
38
|
Inspired by: https://gist.github.com/nymous/f138c7f06062b7c43c060bf03759c29e
|
|
37
39
|
"""
|
|
38
40
|
from structlog.stdlib import ProcessorFormatter
|
|
39
41
|
|
|
40
|
-
|
|
42
|
+
global_log_level = get_environment_log_level_as_string()
|
|
41
43
|
|
|
42
44
|
# TODO I don't understand why we can't use a processor stack as-is here. Need to investigate further.
|
|
43
45
|
|
|
46
|
+
# TODO why are we importing this here?
|
|
44
47
|
# Use ProcessorFormatter to format log records using structlog processors
|
|
45
48
|
from .__init__ import get_default_processors
|
|
46
49
|
|
|
47
|
-
|
|
50
|
+
default_processors = get_default_processors(json_logger=json_logger)
|
|
48
51
|
|
|
49
52
|
formatter = ProcessorFormatter(
|
|
50
53
|
processors=[
|
|
51
54
|
# required to strip extra keys that the structlog stdlib bindings add in
|
|
52
55
|
structlog.stdlib.ProcessorFormatter.remove_processors_meta,
|
|
53
|
-
|
|
56
|
+
default_processors[-1]
|
|
54
57
|
if not is_production() and not is_staging()
|
|
55
58
|
# don't use ORJSON here, as the stdlib formatter chain expects a str not a bytes
|
|
56
59
|
else structlog.processors.JSONRenderer(sort_keys=True),
|
|
@@ -61,7 +64,7 @@ def redirect_stdlib_loggers(json_logger: bool):
|
|
|
61
64
|
# https://github.com/hynek/structlog/issues/254
|
|
62
65
|
structlog.stdlib.add_logger_name,
|
|
63
66
|
# omit the renderer so we can implement our own
|
|
64
|
-
*
|
|
67
|
+
*default_processors[:-1],
|
|
65
68
|
],
|
|
66
69
|
)
|
|
67
70
|
|
|
@@ -73,22 +76,29 @@ def redirect_stdlib_loggers(json_logger: bool):
|
|
|
73
76
|
file_handler.setFormatter(formatter)
|
|
74
77
|
return file_handler
|
|
75
78
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
79
|
+
python_log_path = config("PYTHON_LOG_PATH", default=None)
|
|
80
|
+
|
|
81
|
+
# if json_logger and python_log_path:
|
|
82
|
+
|
|
83
|
+
default_handler = (
|
|
84
|
+
logging.FileHandler(python_log_path)
|
|
85
|
+
if python_log_path
|
|
86
|
+
else logging.StreamHandler(sys.stdout)
|
|
87
|
+
)
|
|
88
|
+
default_handler.setLevel(global_log_level)
|
|
89
|
+
default_handler.setFormatter(formatter)
|
|
80
90
|
|
|
81
91
|
# Configure the root logger
|
|
82
92
|
root_logger = logging.getLogger()
|
|
83
|
-
root_logger.setLevel(
|
|
84
|
-
root_logger.handlers = [
|
|
93
|
+
root_logger.setLevel(global_log_level)
|
|
94
|
+
root_logger.handlers = [default_handler]
|
|
85
95
|
|
|
86
96
|
# Disable propagation to avoid duplicate logs
|
|
87
97
|
root_logger.propagate = True
|
|
88
98
|
|
|
89
99
|
# TODO there is a JSON-like format that can be used to configure loggers instead :/
|
|
100
|
+
# we should probably transition to using that format instead of this customized mapping
|
|
90
101
|
std_logging_configuration = {
|
|
91
|
-
"httpcore": {},
|
|
92
102
|
"httpx": {
|
|
93
103
|
"levels": {
|
|
94
104
|
"INFO": "WARNING",
|
|
@@ -99,12 +109,13 @@ def redirect_stdlib_loggers(json_logger: bool):
|
|
|
99
109
|
"INFO": "WARNING",
|
|
100
110
|
}
|
|
101
111
|
},
|
|
112
|
+
# stripe INFO logs are pretty noisy by default
|
|
113
|
+
"stripe": {
|
|
114
|
+
"levels": {
|
|
115
|
+
"INFO": "WARNING",
|
|
116
|
+
}
|
|
117
|
+
},
|
|
102
118
|
}
|
|
103
|
-
|
|
104
|
-
# Merged from silence_loud_loggers - only silence asyncio if not explicitly debugging it
|
|
105
|
-
if not PYTHONASYNCIODEBUG:
|
|
106
|
-
std_logging_configuration["asyncio"] = {"level": "WARNING"}
|
|
107
|
-
|
|
108
119
|
"""
|
|
109
120
|
These loggers either:
|
|
110
121
|
|
|
@@ -116,45 +127,65 @@ def redirect_stdlib_loggers(json_logger: bool):
|
|
|
116
127
|
for a set of standard loggers.
|
|
117
128
|
"""
|
|
118
129
|
|
|
119
|
-
|
|
130
|
+
# TODO do we need this? could be AI slop
|
|
131
|
+
|
|
132
|
+
if not PYTHONASYNCIODEBUG:
|
|
133
|
+
std_logging_configuration["asyncio"] = {"level": "WARNING"}
|
|
134
|
+
|
|
135
|
+
environment_logger_config = get_custom_logger_config()
|
|
120
136
|
|
|
121
137
|
# now, let's handle some loggers that are probably already initialized with a handler
|
|
122
138
|
for logger_name, logger_config in std_logging_configuration.items():
|
|
123
139
|
level_override = None
|
|
124
140
|
|
|
125
|
-
#
|
|
141
|
+
# if we have a level override, use that
|
|
126
142
|
if "level" in logger_config:
|
|
127
143
|
level_override = logger_config["level"]
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
144
|
+
assert isinstance(level_override, str), (
|
|
145
|
+
f"Expected level override for {logger_name} to be a string, got {type(level_override)}"
|
|
146
|
+
)
|
|
147
|
+
# Check if we have a level mapping for the current log level
|
|
148
|
+
elif "levels" in logger_config and global_log_level in logger_config["levels"]:
|
|
149
|
+
level_override = logger_config["levels"][global_log_level]
|
|
150
|
+
|
|
151
|
+
# if a static override exists, only use it if it is lower than the global log level
|
|
152
|
+
if level_override and (
|
|
153
|
+
compare_log_levels(
|
|
154
|
+
level_override,
|
|
155
|
+
global_log_level,
|
|
156
|
+
)
|
|
157
|
+
< 0
|
|
158
|
+
):
|
|
159
|
+
level_override = None
|
|
160
|
+
|
|
161
|
+
handler_for_logger = default_handler
|
|
133
162
|
|
|
134
163
|
# Override with environment-specific config if available
|
|
135
164
|
if logger_name in environment_logger_config:
|
|
136
165
|
env_config = environment_logger_config[logger_name]
|
|
137
166
|
|
|
138
167
|
# if we have a custom path, use that instead
|
|
168
|
+
# right now this is the only handler override type we support
|
|
139
169
|
if "path" in env_config:
|
|
140
170
|
handler_for_logger = handler_for_path(env_config["path"])
|
|
141
171
|
|
|
172
|
+
# if the level is set via dynamic config, always use that
|
|
142
173
|
if "level" in env_config:
|
|
143
174
|
level_override = env_config["level"]
|
|
144
175
|
|
|
145
176
|
reset_stdlib_logger(
|
|
146
177
|
logger_name,
|
|
147
178
|
handler_for_logger,
|
|
148
|
-
level_override,
|
|
179
|
+
level_override or global_log_level,
|
|
149
180
|
)
|
|
150
181
|
|
|
151
182
|
# Handle any additional loggers defined in environment variables
|
|
152
183
|
for logger_name, logger_config in environment_logger_config.items():
|
|
153
|
-
# skip if already configured
|
|
184
|
+
# skip if already configured via the above loop
|
|
154
185
|
if logger_name in std_logging_configuration:
|
|
155
186
|
continue
|
|
156
187
|
|
|
157
|
-
handler_for_logger =
|
|
188
|
+
handler_for_logger = default_handler
|
|
158
189
|
|
|
159
190
|
if "path" in logger_config:
|
|
160
191
|
# if we have a custom path, use that instead
|
|
@@ -163,7 +194,7 @@ def redirect_stdlib_loggers(json_logger: bool):
|
|
|
163
194
|
reset_stdlib_logger(
|
|
164
195
|
logger_name,
|
|
165
196
|
handler_for_logger,
|
|
166
|
-
logger_config.get("level"),
|
|
197
|
+
logger_config.get("level", global_log_level),
|
|
167
198
|
)
|
|
168
199
|
|
|
169
200
|
# TODO do i need to setup exception overrides as well?
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Adapted from:
|
|
3
|
+
- https://github.com/willmcgugan/httpx/blob/973d1ed4e06577d928061092affe8f94def03331/httpx/_utils.py#L231
|
|
4
|
+
- https://github.com/vladmandic/sdnext/blob/d5d857aa961edbc46c9e77e7698f2e60011e7439/installer.py#L154
|
|
5
|
+
|
|
6
|
+
TODO this is not fully integrated into the codebase
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import logging
|
|
10
|
+
import typing
|
|
11
|
+
from functools import partial, partialmethod
|
|
12
|
+
|
|
13
|
+
from structlog._log_levels import NAME_TO_LEVEL
|
|
14
|
+
from structlog._native import LEVEL_TO_FILTERING_LOGGER, _make_filtering_bound_logger
|
|
15
|
+
|
|
16
|
+
from structlog_config.constants import TRACE_LOG_LEVEL
|
|
17
|
+
|
|
18
|
+
# Track if setup has already been called
|
|
19
|
+
_setup_called = False
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
# Stub for type checkers.
|
|
23
|
+
class Logger(logging.Logger):
|
|
24
|
+
def trace(
|
|
25
|
+
self, message: str, *args: typing.Any, **kwargs: typing.Any
|
|
26
|
+
) -> None: ... # pragma: nocover
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
# def trace(self, message: str, *args: typing.Any, **kwargs: typing.Any) -> None:
|
|
30
|
+
# if self.isEnabledFor(TRACE_LOG_LEVEL):
|
|
31
|
+
# self._log(TRACE_LOG_LEVEL, message, args, **kwargs)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def setup_trace() -> None:
|
|
35
|
+
"""Setup TRACE logging level. Safe to call multiple times."""
|
|
36
|
+
global _setup_called
|
|
37
|
+
|
|
38
|
+
if _setup_called:
|
|
39
|
+
return
|
|
40
|
+
|
|
41
|
+
# TODO consider adding warning to check the state of the underlying patched code
|
|
42
|
+
# patch structlog maps to include the additional level
|
|
43
|
+
NAME_TO_LEVEL["trace"] = TRACE_LOG_LEVEL
|
|
44
|
+
LEVEL_TO_FILTERING_LOGGER[TRACE_LOG_LEVEL] = _make_filtering_bound_logger(
|
|
45
|
+
TRACE_LOG_LEVEL
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
logging.TRACE = TRACE_LOG_LEVEL
|
|
49
|
+
logging.addLevelName(TRACE_LOG_LEVEL, "TRACE")
|
|
50
|
+
|
|
51
|
+
if hasattr(logging.Logger, "trace"):
|
|
52
|
+
logging.warning("Logger.trace method already exists, not overriding it")
|
|
53
|
+
else:
|
|
54
|
+
logging.Logger.trace = partialmethod(logging.Logger.log, logging.TRACE)
|
|
55
|
+
|
|
56
|
+
# Check if trace function already exists in logging module
|
|
57
|
+
if hasattr(logging, "trace"):
|
|
58
|
+
logging.warning("logging.trace function already exists, overriding it")
|
|
59
|
+
else:
|
|
60
|
+
logging.trace = partial(logging.log, logging.TRACE)
|
|
61
|
+
|
|
62
|
+
_setup_called = True
|
structlog_config/warnings.py
CHANGED
|
@@ -9,6 +9,8 @@ import structlog
|
|
|
9
9
|
|
|
10
10
|
_original_warnings_showwarning: Any = None
|
|
11
11
|
|
|
12
|
+
warning_logger = structlog.get_logger(logger_name="py.warnings")
|
|
13
|
+
|
|
12
14
|
|
|
13
15
|
def _showwarning(
|
|
14
16
|
message: Warning | str,
|
|
@@ -20,23 +22,11 @@ def _showwarning(
|
|
|
20
22
|
) -> Any:
|
|
21
23
|
"""
|
|
22
24
|
Redirects warnings to structlog so they appear in task logs etc.
|
|
23
|
-
|
|
24
|
-
Implementation of showwarnings which redirects to logging, which will first
|
|
25
|
-
check to see if the file parameter is None. If a file is specified, it will
|
|
26
|
-
delegate to the original warnings implementation of showwarning. Otherwise,
|
|
27
|
-
it will call warnings.formatwarning and will log the resulting string to a
|
|
28
|
-
warnings logger named "py.warnings" with level logging.WARNING.
|
|
29
25
|
"""
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
)
|
|
35
|
-
else:
|
|
36
|
-
log = structlog.get_logger(logger_name="py.warnings")
|
|
37
|
-
log.warning(
|
|
38
|
-
str(message), category=category.__name__, filename=filename, lineno=lineno
|
|
39
|
-
)
|
|
26
|
+
|
|
27
|
+
warning_logger.warning(
|
|
28
|
+
str(message), category=category.__name__, filename=filename, lineno=lineno
|
|
29
|
+
)
|
|
40
30
|
|
|
41
31
|
|
|
42
32
|
def redirect_showwarnings():
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: structlog-config
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Summary: A comprehensive structlog configuration with sensible defaults for development and production environments, featuring context management, exception formatting, and path prettification.
|
|
5
|
+
Keywords: logging,structlog,json-logging,structured-logging
|
|
6
|
+
Author: Michael Bianco
|
|
7
|
+
Author-email: Michael Bianco <mike@mikebian.co>
|
|
8
|
+
Requires-Dist: orjson>=3.10.15
|
|
9
|
+
Requires-Dist: python-decouple-typed>=3.11.0
|
|
10
|
+
Requires-Dist: python-ipware>=3.0.0
|
|
11
|
+
Requires-Dist: structlog>=25.2.0
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Project-URL: Repository, https://github.com/iloveitaly/structlog-config
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
|
|
16
|
+
# Opinionated Defaults for Structlog
|
|
17
|
+
|
|
18
|
+
Logging is really important. Getting logging to work well in python feels like black magic: there's a ton of configuration
|
|
19
|
+
across structlog, warnings, std loggers, fastapi + celery context, JSON logging in production, etc that requires lots of
|
|
20
|
+
fiddling and testing to get working. I finally got this working for me in my [project template](https://github.com/iloveitaly/python-starter-template) and extracted this out into a nice package.
|
|
21
|
+
|
|
22
|
+
Here are the main goals:
|
|
23
|
+
|
|
24
|
+
* High performance JSON logging in production
|
|
25
|
+
* All loggers, even plugin or system loggers, should route through the same formatter
|
|
26
|
+
* Structured logging everywhere
|
|
27
|
+
* Ability to easily set thread-local log context
|
|
28
|
+
* Nice log formatters for stack traces, ORM ([ActiveModel/SQLModel](https://github.com/iloveitaly/activemodel)), etc
|
|
29
|
+
* Ability to log level and output (i.e. file path) *by logger* for easy development debugging
|
|
30
|
+
* If you are using fastapi, structured logging for access logs
|
|
31
|
+
|
|
32
|
+
## Usage
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
from structlog_config import configure_logger
|
|
36
|
+
|
|
37
|
+
log = configure_logger()
|
|
38
|
+
|
|
39
|
+
log.info("the log", key="value")
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## TRACE Logging Level
|
|
43
|
+
|
|
44
|
+
This package adds support for a custom `TRACE` logging level (level 5) that's even more verbose than `DEBUG`. This is useful for extremely detailed debugging scenarios.
|
|
45
|
+
|
|
46
|
+
The `TRACE` level is automatically set up when you call `configure_logger()`. You can use it like any other logging level:
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
import logging
|
|
50
|
+
from structlog_config import configure_logger
|
|
51
|
+
|
|
52
|
+
log = configure_logger()
|
|
53
|
+
|
|
54
|
+
# Using structlog
|
|
55
|
+
log.info("This is info")
|
|
56
|
+
log.debug("This is debug")
|
|
57
|
+
log.trace("This is trace") # Most verbose
|
|
58
|
+
|
|
59
|
+
# Using stdlib logging
|
|
60
|
+
logging.trace("Module-level trace message")
|
|
61
|
+
logger = logging.getLogger(__name__)
|
|
62
|
+
logger.trace("Instance trace message")
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Set the log level to TRACE using the environment variable:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
LOG_LEVEL=TRACE
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Stdlib Log Management
|
|
72
|
+
|
|
73
|
+
By default, all stdlib loggers are:
|
|
74
|
+
|
|
75
|
+
1. Given the same global logging level, with some default adjustments for noisy loggers (looking at you, `httpx`)
|
|
76
|
+
2. Use a structlog formatter (you get structured logging, context, etc in any stdlib logger calls)
|
|
77
|
+
3. The root processor is overwritten so any child loggers created after initialization will use the same formatter
|
|
78
|
+
|
|
79
|
+
You can customize loggers by name (i.e. the name used in `logging.getLogger(__name__)`) using ENV variables.
|
|
80
|
+
|
|
81
|
+
For example, if you wanted to [mimic `OPENAI_LOG` functionality](https://github.com/openai/openai-python/blob/de7c0e2d9375d042a42e3db6c17e5af9a5701a99/src/openai/_utils/_logs.py#L16):
|
|
82
|
+
|
|
83
|
+
* `LOG_LEVEL_OPENAI=DEBUG`
|
|
84
|
+
* `LOG_PATH_OPENAI=tmp/openai.log`
|
|
85
|
+
* `LOG_LEVEL_HTTPX=DEBUG`
|
|
86
|
+
* `LOG_PATH_HTTPX=tmp/openai.log`
|
|
87
|
+
|
|
88
|
+
## FastAPI Access Logger
|
|
89
|
+
|
|
90
|
+
Structured, simple access log with request timing to replace the default fastapi access log. Why?
|
|
91
|
+
|
|
92
|
+
1. It's less verbose
|
|
93
|
+
2. Uses structured logging params instead of string interpolation
|
|
94
|
+
3. debug level logs any static assets
|
|
95
|
+
|
|
96
|
+
Here's how to use it:
|
|
97
|
+
|
|
98
|
+
1. [Disable fastapi's default logging.](https://github.com/iloveitaly/python-starter-template/blob/f54cb47d8d104987f2e4a668f9045a62e0d6818a/main.py#L55-L56)
|
|
99
|
+
2. [Add the middleware to your FastAPI app.](https://github.com/iloveitaly/python-starter-template/blob/f54cb47d8d104987f2e4a668f9045a62e0d6818a/app/routes/middleware/__init__.py#L63-L65)
|
|
100
|
+
|
|
101
|
+
## iPython
|
|
102
|
+
|
|
103
|
+
Often it's helpful to update logging level within an iPython session. You can do this and make sure all loggers pick up on it.
|
|
104
|
+
|
|
105
|
+
```
|
|
106
|
+
%env LOG_LEVEL=DEBUG
|
|
107
|
+
from structlog_config import configure_logger
|
|
108
|
+
configure_logger()
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
## Related Projects
|
|
112
|
+
|
|
113
|
+
* https://github.com/underyx/structlog-pretty
|
|
114
|
+
* https://pypi.org/project/httpx-structlog/
|
|
115
|
+
|
|
116
|
+
## References
|
|
117
|
+
|
|
118
|
+
General logging:
|
|
119
|
+
|
|
120
|
+
- https://github.com/replicate/cog/blob/2e57549e18e044982bd100e286a1929f50880383/python/cog/logging.py#L20
|
|
121
|
+
- https://github.com/apache/airflow/blob/4280b83977cd5a53c2b24143f3c9a6a63e298acc/task_sdk/src/airflow/sdk/log.py#L187
|
|
122
|
+
- https://github.com/kiwicom/structlog-sentry
|
|
123
|
+
- https://github.com/jeremyh/datacube-explorer/blob/b289b0cde0973a38a9d50233fe0fff00e8eb2c8e/cubedash/logs.py#L40C21-L40C42
|
|
124
|
+
- https://stackoverflow.com/questions/76256249/logging-in-the-open-ai-python-library/78214464#78214464
|
|
125
|
+
- https://github.com/openai/openai-python/blob/de7c0e2d9375d042a42e3db6c17e5af9a5701a99/src/openai/_utils/_logs.py#L16
|
|
126
|
+
- https://www.python-httpx.org/logging/
|
|
127
|
+
|
|
128
|
+
FastAPI access logger:
|
|
129
|
+
|
|
130
|
+
- https://github.com/iloveitaly/fastapi-logger/blob/main/fastapi_structlog/middleware/access_log.py#L70
|
|
131
|
+
- https://github.com/fastapiutils/fastapi-utils/blob/master/fastapi_utils/timing.py
|
|
132
|
+
- https://pypi.org/project/fastapi-structlog/
|
|
133
|
+
- https://pypi.org/project/asgi-correlation-id/
|
|
134
|
+
- https://gist.github.com/nymous/f138c7f06062b7c43c060bf03759c29e
|
|
135
|
+
- https://github.com/sharu1204/fastapi-structlog/blob/master/app/main.py
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
structlog_config/__init__.py,sha256=DyY4x3_dY_hPNbS1aM7JRCGadTa1dYDIPzgrHu3AP68,6733
|
|
2
|
+
structlog_config/constants.py,sha256=O1nPnB29yZdqqaI7aeTUrimA3LOtA5WpP6BGPLWJvj8,510
|
|
3
|
+
structlog_config/env_config.py,sha256=_EJO0rgAKndRPSh4wuBaH3bui9F3nIpn8FaEkjAjZso,1737
|
|
4
|
+
structlog_config/environments.py,sha256=JpZYVVDGxEf1EaKdPdn6Jo-4wJK6SqF0ueFl7e2TBvI,612
|
|
5
|
+
structlog_config/fastapi_access_logger.py,sha256=REwchxGb5WHbiQi2zYtBPfkGHk-NvCKM2pVwJlwG5H8,5464
|
|
6
|
+
structlog_config/formatters.py,sha256=ll0Y0QeRs1DMmD-ft1n1zA4Vn2STRSK-mOrczYB2OjE,5898
|
|
7
|
+
structlog_config/levels.py,sha256=z1fTpvCCbAwcFK2k7rHWh_p-FqfFh4yIWCTZ1MNf_4U,993
|
|
8
|
+
structlog_config/packages.py,sha256=asxrzLR-iRYAbkoSYutyTdIRcruTjHgkzfe2pjm2VFM,519
|
|
9
|
+
structlog_config/stdlib_logging.py,sha256=Wnn59oRBIqn708CpR-akqVcG9ccSfCMLh56_7wxZRH0,7350
|
|
10
|
+
structlog_config/trace.py,sha256=dBaSynxmw4Wg79wSHqYEMoByvv--v_oQw61dRdg4xUI,2016
|
|
11
|
+
structlog_config/warnings.py,sha256=gKEcuHWqH0BaKitJtQkv-uJ0Z3uCH5nn6k8qpqjR-RM,998
|
|
12
|
+
structlog_config-0.4.0.dist-info/WHEEL,sha256=4n27za1eEkOnA7dNjN6C5-O2rUiw6iapszm14Uj-Qmk,79
|
|
13
|
+
structlog_config-0.4.0.dist-info/METADATA,sha256=R2XtpZuBtPdum5qHcmmCoXbSFoQPm0iv4VQcAX0I7zI,5472
|
|
14
|
+
structlog_config-0.4.0.dist-info/RECORD,,
|
|
@@ -1,62 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: structlog-config
|
|
3
|
-
Version: 0.1.0
|
|
4
|
-
Summary: A comprehensive structlog configuration with sensible defaults for development and production environments, featuring context management, exception formatting, and path prettification.
|
|
5
|
-
Project-URL: Repository, https://github.com/iloveitaly/structlog-config
|
|
6
|
-
Author-email: Michael Bianco <mike@mikebian.co>
|
|
7
|
-
Keywords: json-logging,logging,structlog,structured-logging
|
|
8
|
-
Requires-Python: >=3.10
|
|
9
|
-
Requires-Dist: orjson>=3.10.15
|
|
10
|
-
Requires-Dist: python-decouple-typed>=3.11.0
|
|
11
|
-
Requires-Dist: structlog>=25.2.0
|
|
12
|
-
Description-Content-Type: text/markdown
|
|
13
|
-
|
|
14
|
-
# Structlog Configuration
|
|
15
|
-
|
|
16
|
-
Logging is really important:
|
|
17
|
-
|
|
18
|
-
* High performance JSON logging in production
|
|
19
|
-
* All loggers, even plugin or system loggers, should route through the same formatter
|
|
20
|
-
* Structured logging everywhere
|
|
21
|
-
* Ability to easily set thread-local log context
|
|
22
|
-
|
|
23
|
-
## Stdib Log Management
|
|
24
|
-
|
|
25
|
-
Note that `{LOGGER_NAME}` is the name of the system logger assigned via `logging.getLogger(__name__)`:
|
|
26
|
-
|
|
27
|
-
* `OPENAI_LOG_LEVEL`
|
|
28
|
-
* `OPENAI_LOG_PATH`. Ignored in production.
|
|
29
|
-
|
|
30
|
-
## FastAPI Access Logger
|
|
31
|
-
|
|
32
|
-
Structured, simple access log with request timing to replace the default fastapi access log. Why?
|
|
33
|
-
|
|
34
|
-
1. It's less verbose
|
|
35
|
-
2. Uses structured logging params instead of string interpolation
|
|
36
|
-
3. debug level logs any static assets
|
|
37
|
-
|
|
38
|
-
Here's how to use it:
|
|
39
|
-
|
|
40
|
-
1. [Disable fastapi's default logging.](https://github.com/iloveitaly/python-starter-template/blob/f54cb47d8d104987f2e4a668f9045a62e0d6818a/main.py#L55-L56)
|
|
41
|
-
2. [Add the middleware to your FastAPI app.](https://github.com/iloveitaly/python-starter-template/blob/f54cb47d8d104987f2e4a668f9045a62e0d6818a/app/routes/middleware/__init__.py#L63-L65)
|
|
42
|
-
|
|
43
|
-
Adapted from:
|
|
44
|
-
|
|
45
|
-
- https://github.com/iloveitaly/fastapi-logger/blob/main/fastapi_structlog/middleware/access_log.py#L70
|
|
46
|
-
- https://github.com/fastapiutils/fastapi-utils/blob/master/fastapi_utils/timing.py
|
|
47
|
-
- https://pypi.org/project/fastapi-structlog/
|
|
48
|
-
- https://pypi.org/project/asgi-correlation-id/
|
|
49
|
-
- https://gist.github.com/nymous/f138c7f06062b7c43c060bf03759c29e
|
|
50
|
-
- https://github.com/sharu1204/fastapi-structlog/blob/master/app/main.py
|
|
51
|
-
|
|
52
|
-
## Related Projects
|
|
53
|
-
|
|
54
|
-
* https://github.com/underyx/structlog-pretty
|
|
55
|
-
* https://pypi.org/project/httpx-structlog/
|
|
56
|
-
|
|
57
|
-
## References
|
|
58
|
-
|
|
59
|
-
- https://github.com/replicate/cog/blob/2e57549e18e044982bd100e286a1929f50880383/python/cog/logging.py#L20
|
|
60
|
-
- https://github.com/apache/airflow/blob/4280b83977cd5a53c2b24143f3c9a6a63e298acc/task_sdk/src/airflow/sdk/log.py#L187
|
|
61
|
-
- https://github.com/kiwicom/structlog-sentry
|
|
62
|
-
- https://github.com/jeremyh/datacube-explorer/blob/b289b0cde0973a38a9d50233fe0fff00e8eb2c8e/cubedash/logs.py#L40C21-L40C42
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
structlog_config/__init__.py,sha256=OA1J4X3oWWPyqu1vojagsCrHmWDahqyFP_tAJJMYpTk,6162
|
|
2
|
-
structlog_config/constants.py,sha256=uwfeIMlu6yzl67dOS_JP427CO-9nyHX1kRyjp-Obb1M,260
|
|
3
|
-
structlog_config/env_config.py,sha256=CEjovBIJWxHtbzeqU2VAZ0SwYl8VKL_ECSgIfBU2Pbs,1738
|
|
4
|
-
structlog_config/environments.py,sha256=JpZYVVDGxEf1EaKdPdn6Jo-4wJK6SqF0ueFl7e2TBvI,612
|
|
5
|
-
structlog_config/fastapi_access_logger.py,sha256=DjO0Gn4zRNxXNBeOiibgwlovyg2dHbUFB2UMUzAE4Iw,3462
|
|
6
|
-
structlog_config/formatters.py,sha256=cprGEjvRFphJixbb0nVCpPn9sfw_Wv4d2vPtKDpM05A,5846
|
|
7
|
-
structlog_config/packages.py,sha256=asxrzLR-iRYAbkoSYutyTdIRcruTjHgkzfe2pjm2VFM,519
|
|
8
|
-
structlog_config/stdlib_logging.py,sha256=hQfX-NpEezqbPyvfw-F95i5-i3-zoaAvaWzSLEjsggM,6097
|
|
9
|
-
structlog_config/warnings.py,sha256=c74VRLxhx7jW96vkYfYwrKkGOaqQLLIfKQuaeB7i4n0,1594
|
|
10
|
-
structlog_config-0.1.0.dist-info/METADATA,sha256=uvFkIiX-qnlT0it-Zp1rJ0vb_VAMUsEP_HXIdwlMruM,2654
|
|
11
|
-
structlog_config-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
12
|
-
structlog_config-0.1.0.dist-info/RECORD,,
|