pylogkit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pylogkit/__init__.py +68 -0
- pylogkit/_constants.py +31 -0
- pylogkit/config.py +156 -0
- pylogkit/filters.py +207 -0
- pylogkit/formatters.py +125 -0
- pylogkit/handlers.py +228 -0
- pylogkit/py.typed +0 -0
- pylogkit-0.1.0.dist-info/METADATA +341 -0
- pylogkit-0.1.0.dist-info/RECORD +11 -0
- pylogkit-0.1.0.dist-info/WHEEL +4 -0
- pylogkit-0.1.0.dist-info/licenses/LICENSE +21 -0
pylogkit/__init__.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""pylogkit — structured logging with Slack integration, rate limiting, and deduplication."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
|
|
5
|
+
try:
|
|
6
|
+
__version__ = version("pylogkit")
|
|
7
|
+
except PackageNotFoundError:
|
|
8
|
+
__version__ = "0.0.0"
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
|
|
12
|
+
from .config import setup_logging
|
|
13
|
+
from .filters import ContextFilter, ContextVarsFilter, DeduplicationFilter, RateLimitFilter
|
|
14
|
+
from .formatters import ColoredFormatter, JsonFormatter
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"setup_logging",
|
|
18
|
+
"get_logger",
|
|
19
|
+
"JsonFormatter",
|
|
20
|
+
"ColoredFormatter",
|
|
21
|
+
"ContextFilter",
|
|
22
|
+
"ContextVarsFilter",
|
|
23
|
+
"RateLimitFilter",
|
|
24
|
+
"DeduplicationFilter",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
# Library best practice: NullHandler so library consumers don't get warnings
|
|
28
|
+
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def get_logger(name: str, **context) -> logging.Logger:
|
|
32
|
+
"""Get a logger with optional context fields.
|
|
33
|
+
|
|
34
|
+
Args:
|
|
35
|
+
name: Logger name (typically ``__name__``).
|
|
36
|
+
**context: Static fields added to every record from this logger.
|
|
37
|
+
|
|
38
|
+
If called again with the same logger name, the context is updated
|
|
39
|
+
(merged with existing fields).
|
|
40
|
+
|
|
41
|
+
Returns:
|
|
42
|
+
Configured ``logging.Logger`` instance.
|
|
43
|
+
|
|
44
|
+
Example::
|
|
45
|
+
|
|
46
|
+
from pylogkit import get_logger
|
|
47
|
+
|
|
48
|
+
logger = get_logger(__name__, pipeline="laba_czech")
|
|
49
|
+
logger.info("Deal processed", extra={"deal_id": "123"})
|
|
50
|
+
"""
|
|
51
|
+
log = logging.getLogger(name)
|
|
52
|
+
if context:
|
|
53
|
+
# Find existing ContextFilter or create a new one
|
|
54
|
+
for f in log.filters:
|
|
55
|
+
if isinstance(f, ContextFilter):
|
|
56
|
+
f.update(**context)
|
|
57
|
+
return log
|
|
58
|
+
log.addFilter(ContextFilter(**context))
|
|
59
|
+
return log
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# Lazy import — only loaded when explicitly requested
|
|
63
|
+
def __getattr__(name: str):
|
|
64
|
+
if name == "SlackHandler":
|
|
65
|
+
from .handlers import SlackHandler
|
|
66
|
+
|
|
67
|
+
return SlackHandler
|
|
68
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
pylogkit/_constants.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Shared constants for pylogkit."""
|
|
2
|
+
|
|
3
|
+
# Standard logging record attributes to exclude when collecting extra fields.
|
|
4
|
+
# Used by formatters and handlers to identify user-supplied fields.
|
|
5
|
+
BUILTIN_RECORD_ATTRS: frozenset[str] = frozenset(
|
|
6
|
+
{
|
|
7
|
+
"args",
|
|
8
|
+
"asctime",
|
|
9
|
+
"created",
|
|
10
|
+
"exc_info",
|
|
11
|
+
"exc_text",
|
|
12
|
+
"filename",
|
|
13
|
+
"funcName",
|
|
14
|
+
"levelname",
|
|
15
|
+
"levelno",
|
|
16
|
+
"lineno",
|
|
17
|
+
"message",
|
|
18
|
+
"module",
|
|
19
|
+
"msecs",
|
|
20
|
+
"msg",
|
|
21
|
+
"name",
|
|
22
|
+
"pathname",
|
|
23
|
+
"process",
|
|
24
|
+
"processName",
|
|
25
|
+
"relativeCreated",
|
|
26
|
+
"stack_info",
|
|
27
|
+
"taskName",
|
|
28
|
+
"thread",
|
|
29
|
+
"threadName",
|
|
30
|
+
}
|
|
31
|
+
)
|
pylogkit/config.py
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"""One-call logging setup for applications."""
|
|
2
|
+
|
|
3
|
+
import contextvars
|
|
4
|
+
import logging
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def setup_logging(
|
|
9
|
+
level: str = "INFO",
|
|
10
|
+
service_name: str | None = None,
|
|
11
|
+
json_format: bool | None = None,
|
|
12
|
+
slack_token: str | None = None,
|
|
13
|
+
slack_channel: str | None = None,
|
|
14
|
+
slack_webhook_url: str | None = None,
|
|
15
|
+
slack_level: str = "ERROR",
|
|
16
|
+
slack_rate_limit: int = 1,
|
|
17
|
+
slack_rate_period: float = 60.0,
|
|
18
|
+
slack_dedupe_window: float = 300.0,
|
|
19
|
+
extra_context: dict[str, str] | None = None,
|
|
20
|
+
loggers: dict[str, str] | None = None,
|
|
21
|
+
ctx_var: contextvars.ContextVar | None = None,
|
|
22
|
+
) -> None:
|
|
23
|
+
"""Configure logging for the application in one call.
|
|
24
|
+
|
|
25
|
+
Args:
|
|
26
|
+
level: Root log level (default: INFO).
|
|
27
|
+
service_name: Service identifier added to every log record.
|
|
28
|
+
json_format: Force JSON (True) or colored (False) output.
|
|
29
|
+
If None, auto-detects: JSON when not a TTY, colored otherwise.
|
|
30
|
+
slack_token: Slack Bot token for error notifications.
|
|
31
|
+
slack_channel: Slack channel for error notifications.
|
|
32
|
+
slack_webhook_url: Slack incoming webhook URL (alternative to token).
|
|
33
|
+
slack_level: Minimum level for Slack notifications (default: ERROR).
|
|
34
|
+
slack_rate_limit: Max Slack messages per rate_period (default: 1).
|
|
35
|
+
slack_rate_period: Rate limit window in seconds (default: 60).
|
|
36
|
+
slack_dedupe_window: Deduplication window in seconds (default: 300).
|
|
37
|
+
extra_context: Static fields added to every log record.
|
|
38
|
+
loggers: Per-logger level overrides, e.g. ``{"httpx": "WARNING"}``.
|
|
39
|
+
ctx_var: A ``ContextVar[dict]`` for request-scoped fields.
|
|
40
|
+
If provided, a ``ContextVarsFilter`` is added to all handlers.
|
|
41
|
+
|
|
42
|
+
Example::
|
|
43
|
+
|
|
44
|
+
from pylogkit import setup_logging
|
|
45
|
+
|
|
46
|
+
setup_logging(
|
|
47
|
+
level="INFO",
|
|
48
|
+
service_name="my-api",
|
|
49
|
+
slack_token=os.environ.get("SLACK_BOT_TOKEN"),
|
|
50
|
+
slack_channel="#errors",
|
|
51
|
+
loggers={"httpx": "WARNING", "sqlalchemy": "WARNING"},
|
|
52
|
+
)
|
|
53
|
+
"""
|
|
54
|
+
from .filters import ContextFilter, ContextVarsFilter
|
|
55
|
+
from .formatters import ColoredFormatter, JsonFormatter
|
|
56
|
+
|
|
57
|
+
root = logging.getLogger()
|
|
58
|
+
|
|
59
|
+
# Prevent duplicate setup
|
|
60
|
+
if any(getattr(h, "_pylogkit", False) for h in root.handlers):
|
|
61
|
+
return
|
|
62
|
+
|
|
63
|
+
resolved_level = getattr(logging, level.upper(), logging.INFO)
|
|
64
|
+
root.setLevel(resolved_level)
|
|
65
|
+
|
|
66
|
+
# ── Console handler ──────────────────────────────────
|
|
67
|
+
use_json = json_format if json_format is not None else not sys.stdout.isatty()
|
|
68
|
+
|
|
69
|
+
console = logging.StreamHandler(sys.stdout)
|
|
70
|
+
console.setLevel(resolved_level)
|
|
71
|
+
console._pylogkit = True # type: ignore[attr-defined]
|
|
72
|
+
|
|
73
|
+
if use_json:
|
|
74
|
+
console.setFormatter(JsonFormatter())
|
|
75
|
+
else:
|
|
76
|
+
console.setFormatter(ColoredFormatter())
|
|
77
|
+
|
|
78
|
+
# Add context filter (static fields)
|
|
79
|
+
context = {"service": service_name} if service_name else {}
|
|
80
|
+
if extra_context:
|
|
81
|
+
context.update(extra_context)
|
|
82
|
+
if context:
|
|
83
|
+
console.addFilter(ContextFilter(**context))
|
|
84
|
+
|
|
85
|
+
# Add contextvars filter (request-scoped fields)
|
|
86
|
+
if ctx_var is not None:
|
|
87
|
+
console.addFilter(ContextVarsFilter(ctx_var))
|
|
88
|
+
|
|
89
|
+
root.addHandler(console)
|
|
90
|
+
|
|
91
|
+
# ── Slack handler ────────────────────────────────────
|
|
92
|
+
if slack_token or slack_webhook_url:
|
|
93
|
+
_setup_slack_handler(
|
|
94
|
+
token=slack_token,
|
|
95
|
+
channel=slack_channel,
|
|
96
|
+
webhook_url=slack_webhook_url,
|
|
97
|
+
level=slack_level,
|
|
98
|
+
rate_limit=slack_rate_limit,
|
|
99
|
+
rate_period=slack_rate_period,
|
|
100
|
+
dedupe_window=slack_dedupe_window,
|
|
101
|
+
context=context,
|
|
102
|
+
ctx_var=ctx_var,
|
|
103
|
+
service_name=service_name,
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
# ── Per-logger levels ────────────────────────────────
|
|
107
|
+
if loggers:
|
|
108
|
+
for name, lvl in loggers.items():
|
|
109
|
+
logging.getLogger(name).setLevel(getattr(logging, lvl.upper(), logging.INFO))
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _setup_slack_handler(
|
|
113
|
+
token: str | None,
|
|
114
|
+
channel: str | None,
|
|
115
|
+
webhook_url: str | None,
|
|
116
|
+
level: str,
|
|
117
|
+
rate_limit: int,
|
|
118
|
+
rate_period: float,
|
|
119
|
+
dedupe_window: float,
|
|
120
|
+
context: dict[str, str],
|
|
121
|
+
ctx_var: contextvars.ContextVar | None = None,
|
|
122
|
+
service_name: str | None = None,
|
|
123
|
+
) -> None:
|
|
124
|
+
"""Set up Slack handler with rate limiting and deduplication."""
|
|
125
|
+
from .filters import ContextFilter, ContextVarsFilter, DeduplicationFilter, RateLimitFilter
|
|
126
|
+
from .handlers import SlackHandler
|
|
127
|
+
|
|
128
|
+
try:
|
|
129
|
+
handler = SlackHandler(
|
|
130
|
+
token=token,
|
|
131
|
+
channel=channel,
|
|
132
|
+
webhook_url=webhook_url,
|
|
133
|
+
level=getattr(logging, level.upper(), logging.ERROR),
|
|
134
|
+
service_name=service_name,
|
|
135
|
+
)
|
|
136
|
+
handler._pylogkit = True # type: ignore[attr-defined]
|
|
137
|
+
|
|
138
|
+
# Rate limit: prevent Slack spam
|
|
139
|
+
handler.addFilter(RateLimitFilter(rate=rate_limit, period=rate_period))
|
|
140
|
+
|
|
141
|
+
# Deduplication: suppress identical errors
|
|
142
|
+
if dedupe_window > 0:
|
|
143
|
+
handler.addFilter(DeduplicationFilter(window=dedupe_window))
|
|
144
|
+
|
|
145
|
+
# Context
|
|
146
|
+
if context:
|
|
147
|
+
handler.addFilter(ContextFilter(**context))
|
|
148
|
+
|
|
149
|
+
# Contextvars
|
|
150
|
+
if ctx_var is not None:
|
|
151
|
+
handler.addFilter(ContextVarsFilter(ctx_var))
|
|
152
|
+
|
|
153
|
+
logging.getLogger().addHandler(handler)
|
|
154
|
+
|
|
155
|
+
except Exception as e:
|
|
156
|
+
print(f"[pylogkit] Failed to initialize SlackHandler: {e}", file=sys.stderr)
|
pylogkit/filters.py
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
"""Log filters: context injection, rate limiting, deduplication, and contextvars."""
|
|
2
|
+
|
|
3
|
+
import contextvars
|
|
4
|
+
import logging
|
|
5
|
+
import threading
|
|
6
|
+
import time
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ContextFilter(logging.Filter):
|
|
11
|
+
"""Injects static context fields into every log record.
|
|
12
|
+
|
|
13
|
+
Args:
|
|
14
|
+
**context: Key-value pairs to add to each record.
|
|
15
|
+
|
|
16
|
+
Example::
|
|
17
|
+
|
|
18
|
+
handler.addFilter(ContextFilter(service="my-api", env="production"))
|
|
19
|
+
# Every record will have record.service and record.env
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
def __init__(self, **context: Any):
|
|
23
|
+
super().__init__()
|
|
24
|
+
self._context = context
|
|
25
|
+
|
|
26
|
+
def filter(self, record: logging.LogRecord) -> bool:
|
|
27
|
+
for key, value in self._context.items():
|
|
28
|
+
if not hasattr(record, key):
|
|
29
|
+
setattr(record, key, value)
|
|
30
|
+
return True
|
|
31
|
+
|
|
32
|
+
def update(self, **context: Any) -> None:
|
|
33
|
+
"""Update or add context fields."""
|
|
34
|
+
self._context.update(context)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class ContextVarsFilter(logging.Filter):
|
|
38
|
+
"""Injects context from ``contextvars.ContextVar`` into log records.
|
|
39
|
+
|
|
40
|
+
Useful for request-scoped context (request_id, user_id, etc.)
|
|
41
|
+
without passing ``extra={}`` to every log call.
|
|
42
|
+
|
|
43
|
+
Args:
|
|
44
|
+
ctx_var: A ``ContextVar`` holding a dict of fields to inject.
|
|
45
|
+
|
|
46
|
+
Example::
|
|
47
|
+
|
|
48
|
+
import contextvars
|
|
49
|
+
from pylogkit import ContextVarsFilter
|
|
50
|
+
|
|
51
|
+
log_context: contextvars.ContextVar[dict] = contextvars.ContextVar(
|
|
52
|
+
"log_context", default={}
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
# Attach to handler or logger
|
|
56
|
+
handler.addFilter(ContextVarsFilter(log_context))
|
|
57
|
+
|
|
58
|
+
# In request handler:
|
|
59
|
+
log_context.set({"request_id": "abc-123", "user_id": "42"})
|
|
60
|
+
logger.info("Processing request") # includes request_id and user_id
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
def __init__(self, ctx_var: contextvars.ContextVar):
|
|
64
|
+
super().__init__()
|
|
65
|
+
self._ctx_var = ctx_var
|
|
66
|
+
|
|
67
|
+
def filter(self, record: logging.LogRecord) -> bool:
|
|
68
|
+
ctx = self._ctx_var.get(None)
|
|
69
|
+
if ctx is not None:
|
|
70
|
+
for key, value in ctx.items():
|
|
71
|
+
if not hasattr(record, key):
|
|
72
|
+
setattr(record, key, value)
|
|
73
|
+
return True
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class RateLimitFilter(logging.Filter):
|
|
77
|
+
"""Limits how often the same log message can pass through.
|
|
78
|
+
|
|
79
|
+
Uses a token bucket per unique message. Each unique message is identified
|
|
80
|
+
by its ``msg`` template (before formatting) + logger name + level.
|
|
81
|
+
|
|
82
|
+
Args:
|
|
83
|
+
rate: Maximum messages per ``period`` seconds for each unique message.
|
|
84
|
+
period: Time window in seconds (default: 60).
|
|
85
|
+
|
|
86
|
+
Example::
|
|
87
|
+
|
|
88
|
+
handler.addFilter(RateLimitFilter(rate=1, period=60))
|
|
89
|
+
# Each unique message passes at most once per minute.
|
|
90
|
+
"""
|
|
91
|
+
|
|
92
|
+
_CLEANUP_EVERY = 100 # prune stale keys every N calls
|
|
93
|
+
|
|
94
|
+
def __init__(self, rate: int = 1, period: float = 60.0):
|
|
95
|
+
super().__init__()
|
|
96
|
+
self.rate = rate
|
|
97
|
+
self.period = period
|
|
98
|
+
self._buckets: dict[str, list] = {} # key -> [timestamps]
|
|
99
|
+
self._lock = threading.Lock()
|
|
100
|
+
self._call_count = 0
|
|
101
|
+
|
|
102
|
+
def filter(self, record: logging.LogRecord) -> bool:
|
|
103
|
+
key = f"{record.name}:{record.levelno}:{record.msg}"
|
|
104
|
+
now = time.monotonic()
|
|
105
|
+
|
|
106
|
+
with self._lock:
|
|
107
|
+
self._call_count += 1
|
|
108
|
+
if self._call_count % self._CLEANUP_EVERY == 0:
|
|
109
|
+
self._prune(now)
|
|
110
|
+
|
|
111
|
+
timestamps = self._buckets.get(key)
|
|
112
|
+
if timestamps is None:
|
|
113
|
+
self._buckets[key] = [now]
|
|
114
|
+
return True
|
|
115
|
+
|
|
116
|
+
# Remove expired timestamps
|
|
117
|
+
cutoff = now - self.period
|
|
118
|
+
timestamps[:] = [t for t in timestamps if t > cutoff]
|
|
119
|
+
|
|
120
|
+
if len(timestamps) >= self.rate:
|
|
121
|
+
return False
|
|
122
|
+
|
|
123
|
+
timestamps.append(now)
|
|
124
|
+
return True
|
|
125
|
+
|
|
126
|
+
def _prune(self, now: float) -> None:
|
|
127
|
+
"""Remove keys with only expired timestamps."""
|
|
128
|
+
cutoff = now - self.period
|
|
129
|
+
stale = [k for k, ts in self._buckets.items() if all(t <= cutoff for t in ts)]
|
|
130
|
+
for k in stale:
|
|
131
|
+
del self._buckets[k]
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class DeduplicationFilter(logging.Filter):
|
|
135
|
+
"""Suppresses duplicate log messages within a time window.
|
|
136
|
+
|
|
137
|
+
Two messages are considered duplicates if they have the same:
|
|
138
|
+
- logger name
|
|
139
|
+
- log level
|
|
140
|
+
- message template (``record.msg``, before formatting)
|
|
141
|
+
|
|
142
|
+
On suppression, counts occurrences. When the window expires,
|
|
143
|
+
the next message passes through with the suppressed count
|
|
144
|
+
added as ``record._suppressed_count`` (does not mutate ``record.msg``).
|
|
145
|
+
|
|
146
|
+
Args:
|
|
147
|
+
window: Deduplication window in seconds (default: 300 = 5 min).
|
|
148
|
+
|
|
149
|
+
Example::
|
|
150
|
+
|
|
151
|
+
handler.addFilter(DeduplicationFilter(window=300))
|
|
152
|
+
"""
|
|
153
|
+
|
|
154
|
+
_CLEANUP_EVERY = 100 # prune stale keys every N calls
|
|
155
|
+
|
|
156
|
+
def __init__(self, window: float = 300.0):
|
|
157
|
+
super().__init__()
|
|
158
|
+
self.window = window
|
|
159
|
+
self._seen: dict[str, _DedupeEntry] = {}
|
|
160
|
+
self._lock = threading.Lock()
|
|
161
|
+
self._call_count = 0
|
|
162
|
+
|
|
163
|
+
def filter(self, record: logging.LogRecord) -> bool:
|
|
164
|
+
key = self._make_key(record)
|
|
165
|
+
now = time.monotonic()
|
|
166
|
+
|
|
167
|
+
with self._lock:
|
|
168
|
+
self._call_count += 1
|
|
169
|
+
if self._call_count % self._CLEANUP_EVERY == 0:
|
|
170
|
+
self._prune(now)
|
|
171
|
+
|
|
172
|
+
entry = self._seen.get(key)
|
|
173
|
+
|
|
174
|
+
if entry is None:
|
|
175
|
+
self._seen[key] = _DedupeEntry(first_seen=now, count=1)
|
|
176
|
+
return True
|
|
177
|
+
|
|
178
|
+
# Window expired — let it through, reset
|
|
179
|
+
if now - entry.first_seen >= self.window:
|
|
180
|
+
suppressed = entry.count - 1
|
|
181
|
+
self._seen[key] = _DedupeEntry(first_seen=now, count=1)
|
|
182
|
+
if suppressed > 0:
|
|
183
|
+
record._suppressed_count = suppressed # type: ignore[attr-defined]
|
|
184
|
+
return True
|
|
185
|
+
|
|
186
|
+
# Within window — suppress
|
|
187
|
+
entry.count += 1
|
|
188
|
+
return False
|
|
189
|
+
|
|
190
|
+
def _prune(self, now: float) -> None:
|
|
191
|
+
"""Remove entries with expired windows."""
|
|
192
|
+
stale = [k for k, e in self._seen.items() if now - e.first_seen >= self.window]
|
|
193
|
+
for k in stale:
|
|
194
|
+
del self._seen[k]
|
|
195
|
+
|
|
196
|
+
@staticmethod
|
|
197
|
+
def _make_key(record: logging.LogRecord) -> str:
|
|
198
|
+
"""Create a deduplication key from the record."""
|
|
199
|
+
return f"{record.name}:{record.levelno}:{record.msg}"
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
class _DedupeEntry:
|
|
203
|
+
__slots__ = ("first_seen", "count")
|
|
204
|
+
|
|
205
|
+
def __init__(self, first_seen: float, count: int):
|
|
206
|
+
self.first_seen = first_seen
|
|
207
|
+
self.count = count
|
pylogkit/formatters.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Log formatters: JSON for production, colored for development."""
|
|
2
|
+
|
|
3
|
+
import contextlib
|
|
4
|
+
import logging
|
|
5
|
+
from datetime import UTC, datetime
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from pythonjsonlogger.json import JsonFormatter as BaseJsonFormatter
|
|
9
|
+
|
|
10
|
+
from ._constants import BUILTIN_RECORD_ATTRS
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class JsonFormatter(BaseJsonFormatter):
|
|
14
|
+
"""JSON formatter with clean output for production.
|
|
15
|
+
|
|
16
|
+
Produces structured JSON with:
|
|
17
|
+
- timestamp (ISO 8601 UTC)
|
|
18
|
+
- level, logger, message
|
|
19
|
+
- All extra fields passed via ``extra={}``
|
|
20
|
+
- Exception traceback (if present)
|
|
21
|
+
|
|
22
|
+
Strips verbose internal fields for cleaner output.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def add_fields(
|
|
26
|
+
self,
|
|
27
|
+
log_record: dict[str, Any],
|
|
28
|
+
record: logging.LogRecord,
|
|
29
|
+
message_dict: dict[str, Any],
|
|
30
|
+
) -> None:
|
|
31
|
+
super().add_fields(log_record, record, message_dict)
|
|
32
|
+
|
|
33
|
+
log_record["timestamp"] = datetime.fromtimestamp(record.created, tz=UTC).isoformat()
|
|
34
|
+
log_record["level"] = record.levelname
|
|
35
|
+
log_record["logger"] = record.name
|
|
36
|
+
|
|
37
|
+
# Remove verbose internal fields
|
|
38
|
+
for key in ("levelname", "funcName", "lineno", "pathname", "exc_text", "taskName"):
|
|
39
|
+
log_record.pop(key, None)
|
|
40
|
+
|
|
41
|
+
# Include formatted traceback
|
|
42
|
+
if record.exc_info and not log_record.get("traceback"):
|
|
43
|
+
log_record["traceback"] = self.formatException(record.exc_info)
|
|
44
|
+
|
|
45
|
+
# Include suppressed duplicate count from DeduplicationFilter
|
|
46
|
+
suppressed = getattr(record, "_suppressed_count", 0)
|
|
47
|
+
if suppressed:
|
|
48
|
+
log_record["suppressed_duplicates"] = suppressed
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class ColoredFormatter(logging.Formatter):
|
|
52
|
+
"""Colored formatter for development with extras display.
|
|
53
|
+
|
|
54
|
+
Falls back to plain text if ``colorlog`` is not installed.
|
|
55
|
+
Appends extra fields as ``[key=value ...]`` to the message.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
_MAX_EXTRA_VALUE_LEN = 50
|
|
59
|
+
|
|
60
|
+
def __init__(
|
|
61
|
+
self,
|
|
62
|
+
fmt: str | None = None,
|
|
63
|
+
datefmt: str | None = None,
|
|
64
|
+
use_color: bool = True,
|
|
65
|
+
):
|
|
66
|
+
self._use_color = use_color
|
|
67
|
+
self._inner: logging.Formatter
|
|
68
|
+
|
|
69
|
+
default_fmt = "%(asctime)s %(levelname)-8s %(name)s — %(message)s"
|
|
70
|
+
|
|
71
|
+
if use_color:
|
|
72
|
+
try:
|
|
73
|
+
import colorlog
|
|
74
|
+
|
|
75
|
+
self._inner = colorlog.ColoredFormatter(
|
|
76
|
+
f"%(log_color)s{default_fmt}%(reset)s",
|
|
77
|
+
datefmt=datefmt,
|
|
78
|
+
log_colors={
|
|
79
|
+
"DEBUG": "cyan",
|
|
80
|
+
"INFO": "green",
|
|
81
|
+
"WARNING": "yellow",
|
|
82
|
+
"ERROR": "red",
|
|
83
|
+
"CRITICAL": "red,bg_white",
|
|
84
|
+
},
|
|
85
|
+
)
|
|
86
|
+
except ImportError:
|
|
87
|
+
self._use_color = False
|
|
88
|
+
self._inner = logging.Formatter(fmt or default_fmt, datefmt=datefmt)
|
|
89
|
+
else:
|
|
90
|
+
self._inner = logging.Formatter(fmt or default_fmt, datefmt=datefmt)
|
|
91
|
+
|
|
92
|
+
def format(self, record: logging.LogRecord) -> str:
|
|
93
|
+
# Work on a copy to avoid mutating the original record
|
|
94
|
+
# (important when multiple handlers process the same record)
|
|
95
|
+
record = logging.makeLogRecord(record.__dict__)
|
|
96
|
+
|
|
97
|
+
# Collect extra fields
|
|
98
|
+
extras = {
|
|
99
|
+
k: v
|
|
100
|
+
for k, v in record.__dict__.items()
|
|
101
|
+
if k not in BUILTIN_RECORD_ATTRS and not k.startswith("_")
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
# Format the message first (resolves % args), then append extras
|
|
105
|
+
# This prevents breakage when extra values contain % characters
|
|
106
|
+
if record.args:
|
|
107
|
+
with contextlib.suppress(TypeError, ValueError):
|
|
108
|
+
record.msg = record.msg % record.args
|
|
109
|
+
record.args = None
|
|
110
|
+
|
|
111
|
+
# Append suppressed duplicate count from DeduplicationFilter
|
|
112
|
+
suppressed = getattr(record, "_suppressed_count", 0)
|
|
113
|
+
if suppressed:
|
|
114
|
+
record.msg = f"{record.msg} (suppressed {suppressed} duplicates)"
|
|
115
|
+
|
|
116
|
+
if extras:
|
|
117
|
+
parts = []
|
|
118
|
+
for k, v in extras.items():
|
|
119
|
+
s = str(v)
|
|
120
|
+
if len(s) > self._MAX_EXTRA_VALUE_LEN:
|
|
121
|
+
s = s[: self._MAX_EXTRA_VALUE_LEN] + "…"
|
|
122
|
+
parts.append(f"{k}={s}")
|
|
123
|
+
record.msg = f"{record.msg} [{' '.join(parts)}]"
|
|
124
|
+
|
|
125
|
+
return self._inner.format(record)
|
pylogkit/handlers.py
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
"""Slack log handler with async dispatch and rate limiting."""
|
|
2
|
+
|
|
3
|
+
import atexit
|
|
4
|
+
import contextlib
|
|
5
|
+
import logging
|
|
6
|
+
import queue
|
|
7
|
+
import threading
|
|
8
|
+
import time
|
|
9
|
+
import traceback
|
|
10
|
+
|
|
11
|
+
from ._constants import BUILTIN_RECORD_ATTRS
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class SlackHandler(logging.Handler):
|
|
15
|
+
"""Sends log records to a Slack channel via ``slack_sdk``.
|
|
16
|
+
|
|
17
|
+
Features:
|
|
18
|
+
- **Non-blocking**: dispatches messages via a background thread + queue.
|
|
19
|
+
- **Built-in rate limiting**: respects Slack's 1 msg/sec limit per channel.
|
|
20
|
+
- **Exponential backoff**: backs off on consecutive Slack failures.
|
|
21
|
+
- **Graceful shutdown**: flushes pending messages on interpreter exit.
|
|
22
|
+
- **Fallback**: logs to stderr if Slack is unreachable.
|
|
23
|
+
|
|
24
|
+
Requires ``slack-sdk``: install with ``pip install pylogkit[slack]``.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
token: Slack Bot OAuth token (``xoxb-...``).
|
|
28
|
+
channel: Slack channel ID or name (e.g. ``#alerts``).
|
|
29
|
+
level: Minimum log level (default: ERROR).
|
|
30
|
+
max_queue_size: Max queued messages before dropping (default: 100).
|
|
31
|
+
webhook_url: Use incoming webhook URL instead of Bot token.
|
|
32
|
+
If provided, ``token`` and ``channel`` are ignored.
|
|
33
|
+
|
|
34
|
+
Example::
|
|
35
|
+
|
|
36
|
+
handler = SlackHandler(token="xoxb-...", channel="#errors")
|
|
37
|
+
handler.setLevel(logging.ERROR)
|
|
38
|
+
logging.getLogger().addHandler(handler)
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
# Slack rate limit: 1 message per second per channel
|
|
42
|
+
_MIN_INTERVAL = 1.1
|
|
43
|
+
# Exponential backoff limits
|
|
44
|
+
_MAX_BACKOFF = 300.0 # 5 minutes max
|
|
45
|
+
_BACKOFF_BASE = 2.0
|
|
46
|
+
|
|
47
|
+
def __init__(
|
|
48
|
+
self,
|
|
49
|
+
token: str | None = None,
|
|
50
|
+
channel: str | None = None,
|
|
51
|
+
level: int = logging.ERROR,
|
|
52
|
+
max_queue_size: int = 100,
|
|
53
|
+
webhook_url: str | None = None,
|
|
54
|
+
service_name: str | None = None,
|
|
55
|
+
):
|
|
56
|
+
super().__init__(level)
|
|
57
|
+
|
|
58
|
+
self._webhook_url = webhook_url
|
|
59
|
+
self._token = token
|
|
60
|
+
self._channel = channel
|
|
61
|
+
self._service_name = service_name
|
|
62
|
+
|
|
63
|
+
if not webhook_url and not token:
|
|
64
|
+
raise ValueError("Either 'token' or 'webhook_url' must be provided")
|
|
65
|
+
if not webhook_url and not channel:
|
|
66
|
+
raise ValueError("'channel' is required when using token auth")
|
|
67
|
+
|
|
68
|
+
self._queue: queue.Queue = queue.Queue(maxsize=max_queue_size)
|
|
69
|
+
self._shutdown = threading.Event()
|
|
70
|
+
|
|
71
|
+
self._thread = threading.Thread(
|
|
72
|
+
target=self._worker,
|
|
73
|
+
name="pylogkit-slack",
|
|
74
|
+
daemon=True,
|
|
75
|
+
)
|
|
76
|
+
self._thread.start()
|
|
77
|
+
atexit.register(self.close)
|
|
78
|
+
|
|
79
|
+
def emit(self, record: logging.LogRecord) -> None:
|
|
80
|
+
"""Queue the record for async dispatch. Drops if queue is full."""
|
|
81
|
+
with contextlib.suppress(queue.Full):
|
|
82
|
+
self._queue.put_nowait(record)
|
|
83
|
+
|
|
84
|
+
def close(self) -> None:
|
|
85
|
+
"""Flush remaining messages and stop the worker thread."""
|
|
86
|
+
if self._shutdown.is_set():
|
|
87
|
+
return
|
|
88
|
+
self._shutdown.set()
|
|
89
|
+
self._thread.join(timeout=10)
|
|
90
|
+
super().close()
|
|
91
|
+
|
|
92
|
+
def _worker(self) -> None:
|
|
93
|
+
"""Background worker: drain queue and send to Slack."""
|
|
94
|
+
try:
|
|
95
|
+
client = self._create_client()
|
|
96
|
+
except ImportError as e:
|
|
97
|
+
import sys
|
|
98
|
+
|
|
99
|
+
print(f"[pylogkit] SlackHandler worker failed to start: {e}", file=sys.stderr)
|
|
100
|
+
return
|
|
101
|
+
|
|
102
|
+
last_send = 0.0
|
|
103
|
+
consecutive_failures = 0
|
|
104
|
+
|
|
105
|
+
while not self._shutdown.is_set():
|
|
106
|
+
try:
|
|
107
|
+
record = self._queue.get(timeout=1.0)
|
|
108
|
+
except queue.Empty:
|
|
109
|
+
continue
|
|
110
|
+
|
|
111
|
+
# Rate limit
|
|
112
|
+
elapsed = time.monotonic() - last_send
|
|
113
|
+
if elapsed < self._MIN_INTERVAL:
|
|
114
|
+
time.sleep(self._MIN_INTERVAL - elapsed)
|
|
115
|
+
|
|
116
|
+
success = self._send(client, record)
|
|
117
|
+
last_send = time.monotonic()
|
|
118
|
+
self._queue.task_done()
|
|
119
|
+
|
|
120
|
+
if success:
|
|
121
|
+
consecutive_failures = 0
|
|
122
|
+
else:
|
|
123
|
+
consecutive_failures += 1
|
|
124
|
+
backoff = min(
|
|
125
|
+
self._BACKOFF_BASE**consecutive_failures,
|
|
126
|
+
self._MAX_BACKOFF,
|
|
127
|
+
)
|
|
128
|
+
self._shutdown.wait(timeout=backoff)
|
|
129
|
+
|
|
130
|
+
# Flush remaining on shutdown
|
|
131
|
+
while not self._queue.empty():
|
|
132
|
+
try:
|
|
133
|
+
record = self._queue.get_nowait()
|
|
134
|
+
self._send(client, record)
|
|
135
|
+
time.sleep(self._MIN_INTERVAL)
|
|
136
|
+
except queue.Empty:
|
|
137
|
+
break
|
|
138
|
+
|
|
139
|
+
def _create_client(self):
|
|
140
|
+
"""Create Slack client (lazy import to keep slack_sdk optional)."""
|
|
141
|
+
try:
|
|
142
|
+
if self._webhook_url:
|
|
143
|
+
from slack_sdk.webhook import WebhookClient
|
|
144
|
+
|
|
145
|
+
return WebhookClient(url=self._webhook_url)
|
|
146
|
+
else:
|
|
147
|
+
from slack_sdk import WebClient
|
|
148
|
+
|
|
149
|
+
return WebClient(token=self._token)
|
|
150
|
+
except ImportError as err:
|
|
151
|
+
raise ImportError(
|
|
152
|
+
"slack-sdk is required for SlackHandler. Install with: pip install pylogkit[slack]"
|
|
153
|
+
) from err
|
|
154
|
+
|
|
155
|
+
def _send(self, client, record: logging.LogRecord) -> bool:
|
|
156
|
+
"""Send a single record to Slack. Returns True on success."""
|
|
157
|
+
try:
|
|
158
|
+
attachment = self._format_attachment(record)
|
|
159
|
+
|
|
160
|
+
if self._webhook_url:
|
|
161
|
+
client.send(attachments=[attachment])
|
|
162
|
+
else:
|
|
163
|
+
client.chat_postMessage(
|
|
164
|
+
channel=self._channel,
|
|
165
|
+
text=f"[{record.levelname}] {record.getMessage()[:200]}",
|
|
166
|
+
attachments=[attachment],
|
|
167
|
+
)
|
|
168
|
+
return True
|
|
169
|
+
except Exception as e:
|
|
170
|
+
# Never let Slack failures propagate — print to stderr as fallback
|
|
171
|
+
import sys
|
|
172
|
+
|
|
173
|
+
print(
|
|
174
|
+
f"[pylogkit] Failed to send to Slack: {e}",
|
|
175
|
+
file=sys.stderr,
|
|
176
|
+
)
|
|
177
|
+
return False
|
|
178
|
+
|
|
179
|
+
def _format_attachment(self, record: logging.LogRecord) -> dict:
|
|
180
|
+
"""Format log record as a Slack attachment."""
|
|
181
|
+
color_map = {
|
|
182
|
+
logging.WARNING: "warning",
|
|
183
|
+
logging.ERROR: "danger",
|
|
184
|
+
logging.CRITICAL: "#8B0000",
|
|
185
|
+
}
|
|
186
|
+
color = color_map.get(record.levelno, "warning")
|
|
187
|
+
|
|
188
|
+
text = record.getMessage()
|
|
189
|
+
suppressed = getattr(record, "_suppressed_count", 0)
|
|
190
|
+
if suppressed:
|
|
191
|
+
text = f"{text} (suppressed {suppressed} duplicates)"
|
|
192
|
+
if len(text) > 2000:
|
|
193
|
+
text = text[:2000] + "…"
|
|
194
|
+
|
|
195
|
+
fields = [
|
|
196
|
+
{"title": "Level", "value": record.levelname, "short": True},
|
|
197
|
+
{"title": "Logger", "value": record.name, "short": True},
|
|
198
|
+
{"title": "Module", "value": f"{record.pathname}:{record.lineno}", "short": False},
|
|
199
|
+
]
|
|
200
|
+
|
|
201
|
+
# Include extra fields (structured logging context)
|
|
202
|
+
extras = {
|
|
203
|
+
k: v
|
|
204
|
+
for k, v in record.__dict__.items()
|
|
205
|
+
if k not in BUILTIN_RECORD_ATTRS and not k.startswith("_")
|
|
206
|
+
}
|
|
207
|
+
if extras:
|
|
208
|
+
extras_text = "\n".join(f"• *{k}*: {v}" for k, v in extras.items())
|
|
209
|
+
if len(extras_text) > 500:
|
|
210
|
+
extras_text = extras_text[:500] + "\n… (truncated)"
|
|
211
|
+
fields.append({"title": "Context", "value": extras_text, "short": False})
|
|
212
|
+
|
|
213
|
+
# Only include traceback if there's a real exception
|
|
214
|
+
if record.exc_info and record.exc_info[0] is not None:
|
|
215
|
+
tb = "".join(traceback.format_exception(*record.exc_info))
|
|
216
|
+
if len(tb) > 1000:
|
|
217
|
+
tb = tb[:1000] + "\n… (truncated)"
|
|
218
|
+
fields.append({"title": "Traceback", "value": f"```{tb}```", "short": False})
|
|
219
|
+
|
|
220
|
+
attachment: dict = {
|
|
221
|
+
"color": color,
|
|
222
|
+
"text": text,
|
|
223
|
+
"fields": fields,
|
|
224
|
+
"ts": int(record.created),
|
|
225
|
+
}
|
|
226
|
+
if self._service_name:
|
|
227
|
+
attachment["footer"] = self._service_name
|
|
228
|
+
return attachment
|
pylogkit/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pylogkit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Structured logging with Slack integration, rate limiting, and deduplication
|
|
5
|
+
Project-URL: Changelog, https://github.com/analytics-team-global/pylogkit/blob/main/CHANGELOG.md
|
|
6
|
+
Project-URL: Repository, https://github.com/analytics-team-global/pylogkit
|
|
7
|
+
Author: Laba
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Topic :: System :: Logging
|
|
18
|
+
Classifier: Typing :: Typed
|
|
19
|
+
Requires-Python: >=3.11
|
|
20
|
+
Requires-Dist: python-json-logger<4.0,>=2.0
|
|
21
|
+
Provides-Extra: all
|
|
22
|
+
Requires-Dist: colorlog>=6.0; extra == 'all'
|
|
23
|
+
Requires-Dist: slack-sdk>=3.20; extra == 'all'
|
|
24
|
+
Provides-Extra: color
|
|
25
|
+
Requires-Dist: colorlog>=6.0; extra == 'color'
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: colorlog>=6.0; extra == 'dev'
|
|
28
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
29
|
+
Requires-Dist: pytest-cov>=5.0; extra == 'dev'
|
|
30
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
31
|
+
Requires-Dist: ruff>=0.4; extra == 'dev'
|
|
32
|
+
Requires-Dist: slack-sdk>=3.20; extra == 'dev'
|
|
33
|
+
Provides-Extra: slack
|
|
34
|
+
Requires-Dist: slack-sdk>=3.20; extra == 'slack'
|
|
35
|
+
Description-Content-Type: text/markdown
|
|
36
|
+
|
|
37
|
+
# pylogkit
|
|
38
|
+
|
|
39
|
+
[](https://github.com/analytics-team-global/pylogkit/actions/workflows/ci.yml)
|
|
40
|
+
[](https://www.python.org/downloads/)
|
|
41
|
+
[](LICENSE)
|
|
42
|
+
|
|
43
|
+
Structured logging with Slack integration, rate limiting, and deduplication.
|
|
44
|
+
|
|
45
|
+
Built on top of Python's standard `logging` module — no custom APIs to learn.
|
|
46
|
+
|
|
47
|
+
## Installation
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
# Core (JSON + colored formatters)
|
|
51
|
+
pip install pylogkit
|
|
52
|
+
|
|
53
|
+
# With Slack support
|
|
54
|
+
pip install pylogkit[slack]
|
|
55
|
+
|
|
56
|
+
# With colored output
|
|
57
|
+
pip install pylogkit[color]
|
|
58
|
+
|
|
59
|
+
# Everything
|
|
60
|
+
pip install pylogkit[all]
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Or as a git dependency:
|
|
64
|
+
|
|
65
|
+
```toml
|
|
66
|
+
# pyproject.toml
|
|
67
|
+
dependencies = [
|
|
68
|
+
"pylogkit[all] @ git+ssh://git@github.com/analytics-team-global/pylogkit.git",
|
|
69
|
+
]
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## Quick Start
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
import os
|
|
76
|
+
from pylogkit import setup_logging, get_logger
|
|
77
|
+
|
|
78
|
+
setup_logging(
|
|
79
|
+
level="INFO",
|
|
80
|
+
service_name="my-api",
|
|
81
|
+
slack_token=os.environ.get("SLACK_BOT_TOKEN"),
|
|
82
|
+
slack_channel="#errors",
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
logger = get_logger(__name__)
|
|
86
|
+
|
|
87
|
+
logger.info("Server started", extra={"port": 8000})
|
|
88
|
+
logger.error("Request failed", extra={"endpoint": "/users", "status": 500})
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Features
|
|
92
|
+
|
|
93
|
+
### Auto-detect output format
|
|
94
|
+
|
|
95
|
+
- **TTY (development)**: colored human-readable output with extras
|
|
96
|
+
- **Non-TTY (production/Docker)**: structured JSON
|
|
97
|
+
|
|
98
|
+
Force a specific format:
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
setup_logging(json_format=True) # always JSON
|
|
102
|
+
setup_logging(json_format=False) # always colored
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
### Structured logging with extra fields
|
|
106
|
+
|
|
107
|
+
Extra fields are included in both JSON and colored output:
|
|
108
|
+
|
|
109
|
+
```python
|
|
110
|
+
logger.info("Deal processed", extra={
|
|
111
|
+
"deal_id": "123",
|
|
112
|
+
"pipeline": "laba_czech",
|
|
113
|
+
"duration": 1.23,
|
|
114
|
+
})
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
**JSON output:**
|
|
118
|
+
```json
|
|
119
|
+
{"timestamp": "2026-03-18T12:00:00+00:00", "level": "INFO", "logger": "myapp.deals", "message": "Deal processed", "deal_id": "123", "pipeline": "laba_czech", "duration": 1.23}
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
**Colored output:**
|
|
123
|
+
```
|
|
124
|
+
2026-03-18 12:00:00 INFO myapp.deals — Deal processed [deal_id=123 pipeline=laba_czech duration=1.23]
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
### Logger with static context
|
|
128
|
+
|
|
129
|
+
Bind context fields to a logger — they appear in every message:
|
|
130
|
+
|
|
131
|
+
```python
|
|
132
|
+
logger = get_logger(__name__, pipeline="laba_czech", env="production")
|
|
133
|
+
|
|
134
|
+
logger.info("Course synced")
|
|
135
|
+
# All messages from this logger will include pipeline= and env=
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
Calling `get_logger()` again with the same name updates the context:
|
|
139
|
+
|
|
140
|
+
```python
|
|
141
|
+
logger = get_logger(__name__, pipeline="v2")
|
|
142
|
+
# pipeline is now "v2", env remains from the previous call
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
### Request-scoped context with contextvars
|
|
146
|
+
|
|
147
|
+
Inject request-scoped fields (request_id, user_id, etc.) without passing `extra={}` to every log call:
|
|
148
|
+
|
|
149
|
+
```python
|
|
150
|
+
import contextvars
|
|
151
|
+
from pylogkit import setup_logging
|
|
152
|
+
|
|
153
|
+
log_context: contextvars.ContextVar[dict] = contextvars.ContextVar(
|
|
154
|
+
"log_context", default={}
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
setup_logging(
|
|
158
|
+
service_name="my-api",
|
|
159
|
+
ctx_var=log_context,
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
# In your request handler / middleware:
|
|
163
|
+
log_context.set({"request_id": "abc-123", "user_id": "42"})
|
|
164
|
+
logger.info("Processing request")
|
|
165
|
+
# Output includes request_id and user_id automatically
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
Works with async frameworks (FastAPI, aiohttp) — each coroutine gets its own context.
|
|
169
|
+
|
|
170
|
+
### Slack notifications
|
|
171
|
+
|
|
172
|
+
ERROR and CRITICAL logs go to Slack with:
|
|
173
|
+
- Color-coded attachments (yellow/red/dark red)
|
|
174
|
+
- Module and line number
|
|
175
|
+
- Extra fields as "Context" block
|
|
176
|
+
- Formatted traceback
|
|
177
|
+
|
|
178
|
+
```python
|
|
179
|
+
setup_logging(
|
|
180
|
+
slack_token="xoxb-...",
|
|
181
|
+
slack_channel="#alerts",
|
|
182
|
+
slack_level="ERROR", # minimum level (default: ERROR)
|
|
183
|
+
)
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
Or use incoming webhook (simpler, scoped to one channel):
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
setup_logging(
|
|
190
|
+
slack_webhook_url="https://hooks.slack.com/services/T.../B.../xxx",
|
|
191
|
+
)
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
**How it works internally:**
|
|
195
|
+
- Messages are queued and sent from a background thread (non-blocking)
|
|
196
|
+
- Respects Slack's 1 msg/sec rate limit
|
|
197
|
+
- Exponential backoff on consecutive failures (up to 5 min)
|
|
198
|
+
- If queue is full (100 messages) — drops silently, never blocks the app
|
|
199
|
+
- If Slack is unreachable — prints to stderr, never crashes
|
|
200
|
+
- Flushes remaining messages on shutdown
|
|
201
|
+
|
|
202
|
+
### Rate limiting
|
|
203
|
+
|
|
204
|
+
Prevents Slack spam. Each unique message can fire at most N times per period:
|
|
205
|
+
|
|
206
|
+
```python
|
|
207
|
+
setup_logging(
|
|
208
|
+
slack_token="xoxb-...",
|
|
209
|
+
slack_channel="#alerts",
|
|
210
|
+
slack_rate_limit=1, # max 1 message per period (default)
|
|
211
|
+
slack_rate_period=60.0, # per 60 seconds (default)
|
|
212
|
+
)
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
If the same error fires 500 times in a minute — only the first one goes to Slack.
|
|
216
|
+
|
|
217
|
+
### Deduplication
|
|
218
|
+
|
|
219
|
+
Suppresses identical errors within a time window. When the window expires, the next message includes the suppressed count:
|
|
220
|
+
|
|
221
|
+
```python
|
|
222
|
+
setup_logging(
|
|
223
|
+
slack_token="xoxb-...",
|
|
224
|
+
slack_channel="#alerts",
|
|
225
|
+
slack_dedupe_window=300.0, # 5 minutes (default)
|
|
226
|
+
)
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
Example: `"Connection refused (suppressed 47 duplicates)"`
|
|
230
|
+
|
|
231
|
+
Set to `0` to disable deduplication.
|
|
232
|
+
|
|
233
|
+
### Per-logger level overrides
|
|
234
|
+
|
|
235
|
+
Silence noisy third-party libraries:
|
|
236
|
+
|
|
237
|
+
```python
|
|
238
|
+
setup_logging(
|
|
239
|
+
level="INFO",
|
|
240
|
+
loggers={
|
|
241
|
+
"httpx": "WARNING",
|
|
242
|
+
"sqlalchemy.engine": "WARNING",
|
|
243
|
+
"celery": "INFO",
|
|
244
|
+
},
|
|
245
|
+
)
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
### Static context fields
|
|
249
|
+
|
|
250
|
+
Add fields to every log record in the application:
|
|
251
|
+
|
|
252
|
+
```python
|
|
253
|
+
setup_logging(
|
|
254
|
+
service_name="my-api",
|
|
255
|
+
extra_context={
|
|
256
|
+
"env": "production",
|
|
257
|
+
"region": "eu-west-1",
|
|
258
|
+
},
|
|
259
|
+
)
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
## Advanced: using components directly
|
|
263
|
+
|
|
264
|
+
All components can be used independently without `setup_logging()`:
|
|
265
|
+
|
|
266
|
+
```python
|
|
267
|
+
import logging
|
|
268
|
+
from pylogkit import JsonFormatter, ColoredFormatter, RateLimitFilter, DeduplicationFilter
|
|
269
|
+
from pylogkit import SlackHandler # lazy import, requires slack-sdk
|
|
270
|
+
|
|
271
|
+
# Custom handler with JSON
|
|
272
|
+
handler = logging.StreamHandler()
|
|
273
|
+
handler.setFormatter(JsonFormatter())
|
|
274
|
+
|
|
275
|
+
# Slack with custom filters
|
|
276
|
+
slack = SlackHandler(token="xoxb-...", channel="#alerts")
|
|
277
|
+
slack.addFilter(RateLimitFilter(rate=3, period=120))
|
|
278
|
+
slack.addFilter(DeduplicationFilter(window=600))
|
|
279
|
+
|
|
280
|
+
logging.getLogger().addHandler(handler)
|
|
281
|
+
logging.getLogger().addHandler(slack)
|
|
282
|
+
```
|
|
283
|
+
|
|
284
|
+
## Celery integration
|
|
285
|
+
|
|
286
|
+
Prevent Celery from hijacking your logging setup:
|
|
287
|
+
|
|
288
|
+
```python
|
|
289
|
+
from celery.signals import setup_logging as celery_setup_logging
|
|
290
|
+
from pylogkit import setup_logging
|
|
291
|
+
|
|
292
|
+
@celery_setup_logging.connect
|
|
293
|
+
def configure_celery_logging(**kwargs):
|
|
294
|
+
setup_logging(
|
|
295
|
+
service_name="worker",
|
|
296
|
+
slack_token=os.environ.get("SLACK_BOT_TOKEN"),
|
|
297
|
+
slack_channel="#worker-errors",
|
|
298
|
+
)
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
## `setup_logging()` parameters
|
|
302
|
+
|
|
303
|
+
| Parameter | Type | Default | Description |
|
|
304
|
+
|-----------------------|----------------|-----------|--------------------------------------------------------|
|
|
305
|
+
| `level` | `str` | `"INFO"` | Root log level |
|
|
306
|
+
| `service_name` | `str` | `None` | Added to every record as `service` field |
|
|
307
|
+
| `json_format` | `bool` | auto | `True` = JSON, `False` = colored, `None` = auto-detect |
|
|
308
|
+
| `slack_token` | `str` | `None` | Slack Bot token (`xoxb-...`) |
|
|
309
|
+
| `slack_channel` | `str` | `None` | Slack channel (required with token) |
|
|
310
|
+
| `slack_webhook_url` | `str` | `None` | Incoming webhook URL (alternative to token) |
|
|
311
|
+
| `slack_level` | `str` | `"ERROR"` | Minimum level for Slack |
|
|
312
|
+
| `slack_rate_limit` | `int` | `1` | Max messages per period |
|
|
313
|
+
| `slack_rate_period` | `float` | `60.0` | Rate limit window (seconds) |
|
|
314
|
+
| `slack_dedupe_window` | `float` | `300.0` | Deduplication window (seconds), `0` to disable |
|
|
315
|
+
| `extra_context` | `dict` | `None` | Static fields for every record |
|
|
316
|
+
| `loggers` | `dict` | `None` | Per-logger level overrides |
|
|
317
|
+
| `ctx_var` | `ContextVar` | `None` | Request-scoped context variable |
|
|
318
|
+
|
|
319
|
+
## Development
|
|
320
|
+
|
|
321
|
+
```bash
|
|
322
|
+
# Install with dev dependencies
|
|
323
|
+
pip install -e ".[dev]"
|
|
324
|
+
|
|
325
|
+
# Run tests
|
|
326
|
+
pytest
|
|
327
|
+
|
|
328
|
+
# Run tests with coverage
|
|
329
|
+
pytest --cov=pylogkit --cov-report=term-missing
|
|
330
|
+
|
|
331
|
+
# Lint
|
|
332
|
+
ruff check src/ tests/
|
|
333
|
+
ruff format src/ tests/
|
|
334
|
+
|
|
335
|
+
# Type check
|
|
336
|
+
mypy src/pylogkit/
|
|
337
|
+
```
|
|
338
|
+
|
|
339
|
+
## License
|
|
340
|
+
|
|
341
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
pylogkit/__init__.py,sha256=wshcBCG8F8vnSAyQCcJwh73XxUPG1OD-9-DNv-DgIAc,1966
|
|
2
|
+
pylogkit/_constants.py,sha256=WoG1vhYPGJAjH1US0Y3rr0ZcfkKqgPCrXQtVTvDaRo8,696
|
|
3
|
+
pylogkit/config.py,sha256=_yM_sT7f25ODLlsWWxIqjZaGE3aaTqg4COjbcijC0rY,5651
|
|
4
|
+
pylogkit/filters.py,sha256=b8TpfKV6BHussP6nkLd8uftcFMvbatuGchrfxuP3zOM,6507
|
|
5
|
+
pylogkit/formatters.py,sha256=aexexkZ_zQin9yOhKA-HT1wFUqB8QtYrVVj-KAjbt_Y,4294
|
|
6
|
+
pylogkit/handlers.py,sha256=5hPMts8ka_TvWx_TFrzw2-Sa2urRxeo6uRv_xADw1A8,7914
|
|
7
|
+
pylogkit/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
pylogkit-0.1.0.dist-info/METADATA,sha256=kWp-HQ-T5XzSfs8Dgy7IYdV70C7rmAPOEdZtiQbv2Lw,9960
|
|
9
|
+
pylogkit-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
10
|
+
pylogkit-0.1.0.dist-info/licenses/LICENSE,sha256=PAVPx9OpZR0-u06tm8GpkwdUgCsvBbTfp0xxso0jKls,1061
|
|
11
|
+
pylogkit-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Laba
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|