task-logging 0.0.2__tar.gz → 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- task_logging-0.1.0/.github/workflows/ci.yml +41 -0
- {task_logging-0.0.2 → task_logging-0.1.0}/.github/workflows/publish.yml +3 -1
- {task_logging-0.0.2 → task_logging-0.1.0}/.gitignore +4 -0
- task_logging-0.1.0/PKG-INFO +605 -0
- task_logging-0.1.0/PUBLISH.md +314 -0
- task_logging-0.1.0/README.md +586 -0
- task_logging-0.1.0/TODO.md +18 -0
- task_logging-0.1.0/docs/README.md +23 -0
- task_logging-0.0.2/docs/README.md → task_logging-0.1.0/docs/archive-v0.0.1-postgres-design.md +11 -1
- task_logging-0.1.0/docs/design/decorators.md +165 -0
- task_logging-0.1.0/docs/design/json-schema.md +518 -0
- task_logging-0.1.0/docs/design/migrating-from-v0.0.1.md +201 -0
- task_logging-0.1.0/docs/design/stdlib-logging-primer.md +385 -0
- task_logging-0.1.0/docs/design/task-context.md +372 -0
- task_logging-0.1.0/docs/design/why-json-logs.md +239 -0
- {task_logging-0.0.2 → task_logging-0.1.0}/pyproject.toml +27 -3
- task_logging-0.1.0/task_logging/__init__.py +35 -0
- task_logging-0.1.0/task_logging/context.py +171 -0
- task_logging-0.1.0/task_logging/decorators.py +171 -0
- task_logging-0.1.0/task_logging/filters.py +131 -0
- task_logging-0.1.0/task_logging/formatters.py +300 -0
- task_logging-0.1.0/tests/test_context.py +94 -0
- task_logging-0.1.0/tests/test_decorator.py +92 -0
- task_logging-0.1.0/tests/test_formatter.py +182 -0
- task_logging-0.1.0/tests/test_task_logging.py +486 -0
- task_logging-0.1.0/uv.lock +490 -0
- task_logging-0.0.2/.github/workflows/ci.yml +0 -34
- task_logging-0.0.2/PKG-INFO +0 -26
- task_logging-0.0.2/README.md +0 -9
- task_logging-0.0.2/docs/how_to_publish.md +0 -408
- task_logging-0.0.2/error_context.txt +0 -2
- task_logging-0.0.2/error_log.txt +0 -8
- task_logging-0.0.2/task_logging/__init__.py +0 -20
- task_logging-0.0.2/task_logging/models.py +0 -45
- task_logging-0.0.2/task_logging/task_logger.py +0 -438
- task_logging-0.0.2/task_logging/task_logging_database_interface.py +0 -19
- task_logging-0.0.2/tests/simple_task_logger_database.py +0 -33
- task_logging-0.0.2/tests/test_class_func_logger.py +0 -47
- task_logging-0.0.2/tests/test_func_logger.py +0 -38
- task_logging-0.0.2/tests/test_task_logger.py +0 -149
- task_logging-0.0.2/uv.lock +0 -601
- {task_logging-0.0.2 → task_logging-0.1.0}/.pre-commit-config.yaml +0 -0
- {task_logging-0.0.2 → task_logging-0.1.0}/.python-version +0 -0
- {task_logging-0.0.2 → task_logging-0.1.0}/.vscode/settings.json +0 -0
- {task_logging-0.0.2 → task_logging-0.1.0}/LICENSE +0 -0
- {task_logging-0.0.2 → task_logging-0.1.0}/task_logging/py.typed +0 -0
- {task_logging-0.0.2 → task_logging-0.1.0}/tests/__init__.py +0 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# https://docs.astral.sh/uv/guides/integration/github/
|
|
2
|
+
|
|
3
|
+
name: CI
|
|
4
|
+
|
|
5
|
+
on:
|
|
6
|
+
push:
|
|
7
|
+
branches: ["main", "master"]
|
|
8
|
+
pull_request:
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
test:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
|
|
14
|
+
strategy:
|
|
15
|
+
# Run the full matrix even if one version fails — we want to see
|
|
16
|
+
# which versions break, not just the first one.
|
|
17
|
+
fail-fast: false
|
|
18
|
+
matrix:
|
|
19
|
+
python-version: ["3.12", "3.13", "3.14"]
|
|
20
|
+
|
|
21
|
+
steps:
|
|
22
|
+
- name: Checkout
|
|
23
|
+
uses: actions/checkout@v5
|
|
24
|
+
|
|
25
|
+
- name: Install uv
|
|
26
|
+
uses: astral-sh/setup-uv@v7
|
|
27
|
+
|
|
28
|
+
- name: Install Python
|
|
29
|
+
run: uv python install ${{ matrix.python-version }}
|
|
30
|
+
|
|
31
|
+
- name: Sync dependencies
|
|
32
|
+
run: uv sync --group dev --python ${{ matrix.python-version }}
|
|
33
|
+
|
|
34
|
+
- name: Ruff
|
|
35
|
+
run: uv run --python ${{ matrix.python-version }} ruff check .
|
|
36
|
+
|
|
37
|
+
- name: MyPy
|
|
38
|
+
run: uv run --python ${{ matrix.python-version }} mypy .
|
|
39
|
+
|
|
40
|
+
- name: Pytest
|
|
41
|
+
run: uv run --python ${{ matrix.python-version }} pytest --cov=task_logging --cov-report=term-missing
|
|
@@ -25,6 +25,8 @@ jobs:
|
|
|
25
25
|
uses: astral-sh/setup-uv@v7
|
|
26
26
|
|
|
27
27
|
- name: Install Python
|
|
28
|
+
# Build/publish on the lowest supported version; this is a pure-Python
|
|
29
|
+
# wheel so the resulting artifact runs unchanged on 3.12 – 3.14.
|
|
28
30
|
run: uv python install 3.12
|
|
29
31
|
|
|
30
32
|
- name: Sync dependencies
|
|
@@ -37,7 +39,7 @@ jobs:
|
|
|
37
39
|
run: uv run mypy .
|
|
38
40
|
|
|
39
41
|
- name: Pytest
|
|
40
|
-
run: uv run pytest --cov=task_logging --cov-report=term-missing
|
|
42
|
+
run: uv run pytest --cov=task_logging --cov-report=term-missing
|
|
41
43
|
|
|
42
44
|
- name: Build package
|
|
43
45
|
run: uv build
|
|
@@ -0,0 +1,605 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: task-logging
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Task-aware structured logging for distributed Python services.
|
|
5
|
+
Project-URL: Homepage, https://github.com/im-zhong/task-logging
|
|
6
|
+
Project-URL: Repository, https://github.com/im-zhong/task-logging
|
|
7
|
+
Project-URL: Issues, https://github.com/im-zhong/task-logging/issues
|
|
8
|
+
Author-email: zhangzhong <im.zhong@outlook.com>
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: logging,task
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
17
|
+
Requires-Python: <4.0,>=3.12
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
|
|
20
|
+
# Task Logging
|
|
21
|
+
|
|
22
|
+
[](https://www.python.org/downloads/)
|
|
23
|
+
[](LICENSE)
|
|
24
|
+
[](https://pypi.org/project/task-logging/)
|
|
25
|
+
|
|
26
|
+
Task-aware **structured logging** for distributed Python services.
|
|
27
|
+
|
|
28
|
+
The library plugs into Python's stdlib `logging`, lets you bind whatever per-request attrs you want (`task_id`, `user_id`, `trace_id`, …) so they propagate automatically through threads and asyncio tasks, and writes JSON to **stdout**. The container runtime captures stdout, **Grafana Alloy** scrapes it, ships it to **Loki**, and you query it through **Grafana** with LogQL.
|
|
29
|
+
|
|
30
|
+
```
|
|
31
|
+
┌────────────────┐ ┌────────────────┐ ┌────────────────┐
|
|
32
|
+
│ Service A │ │ Service B │ │ Service C │
|
|
33
|
+
│ stdlib logging│ │ stdlib logging│ │ stdlib logging│
|
|
34
|
+
│ + task_logging│ │ + task_logging│ │ + task_logging│
|
|
35
|
+
└───────┬────────┘ └───────┬────────┘ └───────┬────────┘
|
|
36
|
+
│ JSON to stdout │ JSON to stdout │ JSON to stdout
|
|
37
|
+
▼ ▼ ▼
|
|
38
|
+
┌────────────────────────────────────────────────────────────┐
|
|
39
|
+
│ Container runtime (Docker / Kubernetes) captures stdout │
|
|
40
|
+
└──────────────────────────────┬─────────────────────────────┘
|
|
41
|
+
▼
|
|
42
|
+
┌────────────────────────────────────────────────────────────┐
|
|
43
|
+
│ Grafana Alloy (discovers + scrapes containers) │
|
|
44
|
+
└──────────────────────────────┬─────────────────────────────┘
|
|
45
|
+
▼
|
|
46
|
+
┌──────────────┐
|
|
47
|
+
│ Loki │
|
|
48
|
+
└──────┬───────┘
|
|
49
|
+
▼
|
|
50
|
+
┌──────────────┐
|
|
51
|
+
│ Grafana │
|
|
52
|
+
└──────────────┘
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
---
|
|
56
|
+
|
|
57
|
+
## Why this design?
|
|
58
|
+
|
|
59
|
+
- **One central place for logs.** All services write JSON to stdout; the container runtime captures it; Alloy ships it to a single Loki, queryable from one Grafana instance.
|
|
60
|
+
- **Trace a single request across services.** Every log line carries whatever attrs you bound — `task_id`, `service`, `user_id`, anything. Pick any of them in Grafana and follow the request end-to-end.
|
|
61
|
+
- **Third-party logs come along for free.** Because the library plugs into the stdlib root logger, anything that uses `logging` — `requests`, `urllib3`, `boto3`, `sqlalchemy`, your own modules — automatically gets the same JSON pipeline and the same `task_id` tag.
|
|
62
|
+
- **Loki-friendly schema.** `service` / `env` / `level` are low-cardinality (good Loki labels). `task_id` and friends live inside the log line so they don't blow up Loki's stream cardinality.
|
|
63
|
+
- **App stays simple — and 12-factor.** No log files, no rotation knobs, no HTTP, no batching, no retries. Just `print` to stdout (effectively). The platform handles capture, rotation, and shipping. See [12factor.net/logs](https://12factor.net/logs).
|
|
64
|
+
|
|
65
|
+
---
|
|
66
|
+
|
|
67
|
+
## Installation
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
pip install task-logging
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Or with [uv](https://docs.astral.sh/uv/):
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
uv add task-logging
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Requires **Python 3.12+** (tested on 3.12 – 3.14). The library has **zero runtime dependencies** beyond the stdlib.
|
|
80
|
+
|
|
81
|
+
---
|
|
82
|
+
|
|
83
|
+
## Quick Start
|
|
84
|
+
|
|
85
|
+
### 1. Wire up logging once at startup
|
|
86
|
+
|
|
87
|
+
The library deliberately does not provide a one-call setup function — the wiring is six lines of stdlib, and hiding it behind a wrapper would force decisions that belong to you (which handler? which stream? which logger? idempotent or not?). Compose the primitives yourself:
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
import logging
|
|
91
|
+
import sys
|
|
92
|
+
from task_logging import JsonFormatter, TaskLogFilter
|
|
93
|
+
|
|
94
|
+
handler = logging.StreamHandler(sys.stdout)
|
|
95
|
+
handler.setFormatter(JsonFormatter())
|
|
96
|
+
handler.addFilter(TaskLogFilter(global_log_attrs={
|
|
97
|
+
"service": "OrderService", # used as a Loki label
|
|
98
|
+
"env": "prod",
|
|
99
|
+
}))
|
|
100
|
+
|
|
101
|
+
root = logging.getLogger()
|
|
102
|
+
root.addHandler(handler)
|
|
103
|
+
root.setLevel(logging.INFO)
|
|
104
|
+
|
|
105
|
+
# Tame noisy third-party libraries — also stdlib:
|
|
106
|
+
logging.getLogger("urllib3").setLevel(logging.WARNING)
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
> ⚠️ **Attach `TaskLogFilter` to a HANDLER, not to a logger.** Logger-level filters don't see records propagated up from child loggers, so `requests` / `urllib3` / `boto3` records would slip past unenriched. Handler-level filters see every record that reaches the handler, which is what you want.
|
|
110
|
+
|
|
111
|
+
After this, **every** stdlib logger in the process — yours and third-party — writes one line of **JSON** to **stdout**, ready for Alloy.
|
|
112
|
+
|
|
113
|
+
If you need teardown (tests, hot-reloads), it's two lines of stdlib too:
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
root.removeHandler(handler)
|
|
117
|
+
handler.close()
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
Want human-readable output during local development? Pipe through `jq`:
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
python -m myapp | jq
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
That keeps every structured field (`task_id`, `exc_info.locals_dict`, …) visible — a "pretty" formatter would have to drop them.
|
|
127
|
+
|
|
128
|
+
### 2. Use stdlib logging the normal way
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
log = logging.getLogger(__name__)
|
|
132
|
+
log.info("service started")
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
### 3. Tag work with a `task_id`
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
from task_logging import task_log_context
|
|
139
|
+
|
|
140
|
+
def handle_request(req):
|
|
141
|
+
with task_log_context({"task_id": req.id, "user_id": req.user_id}):
|
|
142
|
+
log.info("handling request")
|
|
143
|
+
do_step_1() # logs from here are tagged too
|
|
144
|
+
do_step_2()
|
|
145
|
+
# third-party libs you call inside the block are tagged as well:
|
|
146
|
+
requests.get("https://api.x.com/v1/foo")
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
`task_log_context()` uses Python `contextvars`, so it works correctly across **threads, `asyncio` tasks, and `concurrent.futures` executors** — each concurrent request gets its own isolated context.
|
|
150
|
+
|
|
151
|
+
The library doesn't privilege any particular attr name. Pick whatever keys your domain wants — `task_id`, `request_id`, `trace_id`, `correlation_id` — they all ride through.
|
|
152
|
+
|
|
153
|
+
If you can't use a `with` block (e.g. middleware that binds in a `before_request` hook and unbinds in `after_request`), the same instance also exposes `enter()` / `exit()`:
|
|
154
|
+
|
|
155
|
+
```python
|
|
156
|
+
def before_request(req):
|
|
157
|
+
req.state.log_ctx = task_log_context({"task_id": req.id})
|
|
158
|
+
req.state.log_ctx.enter()
|
|
159
|
+
|
|
160
|
+
def after_request(req):
|
|
161
|
+
req.state.log_ctx.exit()
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
### 4. View it in Grafana
|
|
165
|
+
|
|
166
|
+
Once Alloy → Loki → Grafana is running (see **Deployment** below), this LogQL query gives you everything for one request, across services:
|
|
167
|
+
|
|
168
|
+
```logql
|
|
169
|
+
{env="prod"} | json | task_id="abc-123"
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
---
|
|
173
|
+
|
|
174
|
+
## What gets logged
|
|
175
|
+
|
|
176
|
+
Every record is a single line of JSON with this stable shape. The keys mirror stdlib `LogRecord` attribute names — anyone who knows [stdlib logging](https://docs.python.org/3/library/logging.html#logrecord-attributes) already knows the schema:
|
|
177
|
+
|
|
178
|
+
```json
|
|
179
|
+
{
|
|
180
|
+
"created": 1717839622.503112,
|
|
181
|
+
"levelname": "INFO",
|
|
182
|
+
"name": "billing.settlement",
|
|
183
|
+
"message": "charging account",
|
|
184
|
+
"process": 4321,
|
|
185
|
+
"thread": 140234567890,
|
|
186
|
+
"threadName": "MainThread",
|
|
187
|
+
"module": "settlement",
|
|
188
|
+
"funcName": "charge",
|
|
189
|
+
"pathname": "/app/billing/settlement.py",
|
|
190
|
+
"lineno": 87,
|
|
191
|
+
"exc_info": null,
|
|
192
|
+
|
|
193
|
+
"service": "Billing",
|
|
194
|
+
"env": "prod",
|
|
195
|
+
"task_id": "task-42",
|
|
196
|
+
"user_id": "u-1"
|
|
197
|
+
}
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
The first block mirrors stdlib LogRecord; the second block is whatever **you** bound. The library does not auto-detect anything — `service`, `env`, `task_id`, `user_id` (and `hostname`, if you want it) are all supplied by you via `TaskLogFilter(global_log_attrs=...)` and `task_log_context({...})`.
|
|
201
|
+
|
|
202
|
+
`exc_info` is `null` for normal records and an object for exceptions:
|
|
203
|
+
|
|
204
|
+
```json
|
|
205
|
+
"exc_info": {
|
|
206
|
+
"name": "ZeroDivisionError",
|
|
207
|
+
"details": "division by zero",
|
|
208
|
+
"stack_trace": "Traceback (most recent call last):\n File ...",
|
|
209
|
+
"locals_dict": {"a": "1", "b": "0"}
|
|
210
|
+
}
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
`locals_dict` is a `repr()`-snapshot of the local variables at the deepest stack frame where the exception was raised — invaluable for post-mortem debugging. Disable it with `JsonFormatter(capture_locals=False)` if you're worried about secrets leaking into logs.
|
|
214
|
+
|
|
215
|
+
---
|
|
216
|
+
|
|
217
|
+
## Logging exceptions
|
|
218
|
+
|
|
219
|
+
Inside an `except` block, just call `log.exception()` (or pass `exc_info=True` to any other method). The formatter populates the `exc` field automatically:
|
|
220
|
+
|
|
221
|
+
```python
|
|
222
|
+
def divide(a: int, b: int) -> float:
|
|
223
|
+
return a / b
|
|
224
|
+
|
|
225
|
+
try:
|
|
226
|
+
divide(1, 0)
|
|
227
|
+
except ZeroDivisionError:
|
|
228
|
+
log.exception("division failed")
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
Same goes for raising inside a decorated function — see below.
|
|
232
|
+
|
|
233
|
+
---
|
|
234
|
+
|
|
235
|
+
## The `@log_func_call` decorator
|
|
236
|
+
|
|
237
|
+
For zero-boilerplate enter / exit / timing logs, wrap any function with `log_func_call`. It works on plain functions, instance methods, classmethods, staticmethods — anything — and imposes **no requirements on the surrounding class**.
|
|
238
|
+
|
|
239
|
+
```python
|
|
240
|
+
import logging
|
|
241
|
+
from task_logging import log_func_call
|
|
242
|
+
|
|
243
|
+
log = logging.getLogger(__name__)
|
|
244
|
+
|
|
245
|
+
@log_func_call(log)
|
|
246
|
+
def add(x: int, y: int) -> int:
|
|
247
|
+
return x + y
|
|
248
|
+
|
|
249
|
+
add(2, 3)
|
|
250
|
+
# Logs (as JSON):
|
|
251
|
+
# ENTER add args=(2, 3) kwargs={}
|
|
252
|
+
# EXIT add return=5 cost_ms=0.012
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
It works on methods the same way — no `self._logger` attribute, no setup:
|
|
256
|
+
|
|
257
|
+
```python
|
|
258
|
+
class Service:
|
|
259
|
+
@log_func_call(log)
|
|
260
|
+
def handle(self, payload: dict) -> None:
|
|
261
|
+
...
|
|
262
|
+
# Logs use the qualified name, so methods are easy to tell apart:
|
|
263
|
+
# ENTER Service.handle args=({...},) kwargs={}
|
|
264
|
+
```
|
|
265
|
+
|
|
266
|
+
Omit the logger to auto-resolve `logging.getLogger(func.__module__)` — the stdlib "one logger per module" idiom:
|
|
267
|
+
|
|
268
|
+
```python
|
|
269
|
+
@log_func_call() # uses logging.getLogger(func.__module__)
|
|
270
|
+
def compute() -> int:
|
|
271
|
+
...
|
|
272
|
+
```
|
|
273
|
+
|
|
274
|
+
Override the level if you want:
|
|
275
|
+
|
|
276
|
+
```python
|
|
277
|
+
@log_func_call(log, level=logging.DEBUG)
|
|
278
|
+
def chatty(): ...
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
If the wrapped function raises, `log_func_call` emits a `RAISE` record (with full exception info: stack trace + locals) and re-raises:
|
|
282
|
+
|
|
283
|
+
```
|
|
284
|
+
RAISE add after 0.142ms (exc=ValueError: nope)
|
|
285
|
+
```
|
|
286
|
+
|
|
287
|
+
---
|
|
288
|
+
|
|
289
|
+
## Deployment: Loki + Alloy + Grafana
|
|
290
|
+
|
|
291
|
+
A `docker-compose.yml` with Loki, Grafana, Alloy, and your services is enough to start. Alloy uses **Docker socket discovery** to scrape every container's stdout — no per-service file mounts, no rotation config.
|
|
292
|
+
|
|
293
|
+
### Tag your service containers
|
|
294
|
+
|
|
295
|
+
Alloy reads container labels to figure out the `service` / `env` to attach to logs. Add labels to each app service:
|
|
296
|
+
|
|
297
|
+
```yaml
|
|
298
|
+
services:
|
|
299
|
+
order-service:
|
|
300
|
+
image: my-org/order-service:latest
|
|
301
|
+
labels:
|
|
302
|
+
- "logging=true" # opt this container in
|
|
303
|
+
- "logging.service=OrderService"
|
|
304
|
+
- "logging.env=prod"
|
|
305
|
+
|
|
306
|
+
billing:
|
|
307
|
+
image: my-org/billing:latest
|
|
308
|
+
labels:
|
|
309
|
+
- "logging=true"
|
|
310
|
+
- "logging.service=Billing"
|
|
311
|
+
- "logging.env=prod"
|
|
312
|
+
```
|
|
313
|
+
|
|
314
|
+
(Pick the label keys you like — Alloy lets you map any label to a Loki label. The keys above match the example Alloy config below.)
|
|
315
|
+
|
|
316
|
+
### `docker-compose.yml`
|
|
317
|
+
|
|
318
|
+
```yaml
|
|
319
|
+
services:
|
|
320
|
+
loki:
|
|
321
|
+
image: grafana/loki:3.2.0
|
|
322
|
+
ports: ["3100:3100"]
|
|
323
|
+
command: -config.file=/etc/loki/local-config.yaml
|
|
324
|
+
volumes:
|
|
325
|
+
- loki-data:/loki
|
|
326
|
+
|
|
327
|
+
alloy:
|
|
328
|
+
image: grafana/alloy:latest
|
|
329
|
+
command: run --server.http.listen-addr=0.0.0.0:12345 /etc/alloy/config.alloy
|
|
330
|
+
volumes:
|
|
331
|
+
- ./alloy/config.alloy:/etc/alloy/config.alloy:ro
|
|
332
|
+
# Mount the Docker socket so Alloy can discover and scrape sibling
|
|
333
|
+
# containers' stdout. Read-only is enough.
|
|
334
|
+
- /var/run/docker.sock:/var/run/docker.sock:ro
|
|
335
|
+
ports: ["12345:12345"]
|
|
336
|
+
depends_on: [loki]
|
|
337
|
+
|
|
338
|
+
grafana:
|
|
339
|
+
image: grafana/grafana:latest
|
|
340
|
+
ports: ["3000:3000"]
|
|
341
|
+
environment:
|
|
342
|
+
- GF_AUTH_ANONYMOUS_ENABLED=true
|
|
343
|
+
- GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
|
|
344
|
+
volumes:
|
|
345
|
+
- grafana-data:/var/lib/grafana
|
|
346
|
+
|
|
347
|
+
volumes:
|
|
348
|
+
loki-data:
|
|
349
|
+
grafana-data:
|
|
350
|
+
```
|
|
351
|
+
|
|
352
|
+
### `alloy/config.alloy`
|
|
353
|
+
|
|
354
|
+
```alloy
|
|
355
|
+
// Discover all running containers via the Docker socket.
|
|
356
|
+
discovery.docker "containers" {
|
|
357
|
+
host = "unix:///var/run/docker.sock"
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// Keep only containers explicitly opted in via `logging=true`, and promote
|
|
361
|
+
// their labels into Prometheus-style targets that loki.source.docker can scrape.
|
|
362
|
+
discovery.relabel "containers" {
|
|
363
|
+
targets = discovery.docker.containers.targets
|
|
364
|
+
|
|
365
|
+
// Drop containers that didn't opt in.
|
|
366
|
+
rule {
|
|
367
|
+
source_labels = ["__meta_docker_container_label_logging"]
|
|
368
|
+
regex = "true"
|
|
369
|
+
action = "keep"
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
// Map container labels to Loki labels (low-cardinality only).
|
|
373
|
+
rule {
|
|
374
|
+
source_labels = ["__meta_docker_container_label_logging_service"]
|
|
375
|
+
target_label = "service"
|
|
376
|
+
}
|
|
377
|
+
rule {
|
|
378
|
+
source_labels = ["__meta_docker_container_label_logging_env"]
|
|
379
|
+
target_label = "env"
|
|
380
|
+
}
|
|
381
|
+
// The container name is also useful for distinguishing replicas.
|
|
382
|
+
rule {
|
|
383
|
+
source_labels = ["__meta_docker_container_name"]
|
|
384
|
+
target_label = "container"
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
// Read each opted-in container's stdout/stderr.
|
|
389
|
+
loki.source.docker "containers" {
|
|
390
|
+
host = "unix:///var/run/docker.sock"
|
|
391
|
+
targets = discovery.relabel.containers.output
|
|
392
|
+
forward_to = [loki.process.parse.receiver]
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
// Parse the JSON line. Field names match Python stdlib LogRecord
|
|
396
|
+
// attributes (levelname, created, ...). We rename `levelname` to a
|
|
397
|
+
// shorter `level` Loki label for query ergonomics — that's a labelling
|
|
398
|
+
// decision, not a schema change in the JSON.
|
|
399
|
+
loki.process "parse" {
|
|
400
|
+
forward_to = [loki.write.default.receiver]
|
|
401
|
+
|
|
402
|
+
stage.json {
|
|
403
|
+
expressions = {
|
|
404
|
+
level = "levelname", // pull stdlib's `levelname` out as `level`
|
|
405
|
+
created = "created", // Unix float timestamp from stdlib
|
|
406
|
+
task_id = "task_id", // our own field
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
stage.timestamp {
|
|
411
|
+
source = "created"
|
|
412
|
+
format = "Unix"
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
stage.labels {
|
|
416
|
+
values = { level = "" } // level is a low-cardinality Loki label
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
stage.structured_metadata {
|
|
420
|
+
values = { task_id = "" } // task_id is HIGH-cardinality; never make it a label
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
loki.write "default" {
|
|
425
|
+
endpoint {
|
|
426
|
+
url = "http://loki:3100/loki/api/v1/push"
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
```
|
|
430
|
+
|
|
431
|
+
> **Why `task_id` is in `structured_metadata`, not labels.** Loki indexes by label combinations, so a high-cardinality label like `task_id` would create a new stream per request and crash Loki. `structured_metadata` (Loki ≥ 2.9) gives you fast filtering on high-cardinality fields without that cost.
|
|
432
|
+
|
|
433
|
+
### Bring it up
|
|
434
|
+
|
|
435
|
+
```bash
|
|
436
|
+
docker compose up -d
|
|
437
|
+
```
|
|
438
|
+
|
|
439
|
+
- **Loki**: http://localhost:3100
|
|
440
|
+
- **Alloy UI**: http://localhost:12345
|
|
441
|
+
- **Grafana**: http://localhost:3000
|
|
442
|
+
|
|
443
|
+
Add Loki as a Grafana data source (`http://loki:3100`), then explore:
|
|
444
|
+
|
|
445
|
+
```logql
|
|
446
|
+
# all logs from one service
|
|
447
|
+
{service="OrderService", env="prod"}
|
|
448
|
+
|
|
449
|
+
# follow one request across services
|
|
450
|
+
{env="prod"} | json | task_id="abc-123"
|
|
451
|
+
|
|
452
|
+
# only errors, last hour (the Loki label `level` is set by Alloy from the
|
|
453
|
+
# JSON `levelname` field — see the Alloy config above)
|
|
454
|
+
{env="prod", level=~"ERROR|CRITICAL"}
|
|
455
|
+
|
|
456
|
+
# filter on a structured field bound via task_log_context({"user_id": ...})
|
|
457
|
+
{service="Billing"} | json | user_id="u-42"
|
|
458
|
+
|
|
459
|
+
# filter on a stdlib LogRecord field after `| json`
|
|
460
|
+
{service="Billing"} | json | funcName="charge"
|
|
461
|
+
|
|
462
|
+
# isolate one container replica
|
|
463
|
+
{service="OrderService", container="order-service-2"}
|
|
464
|
+
```
|
|
465
|
+
|
|
466
|
+
### Kubernetes note
|
|
467
|
+
|
|
468
|
+
In a Kubernetes cluster, replace `discovery.docker` with `discovery.kubernetes` and deploy Alloy as a **DaemonSet**. The kubelet already captures every container's stdout into `/var/log/containers/*.log`; Alloy tails those files and uses the pod's labels / annotations (instead of Docker labels) to attach `service` / `env`. Same `loki.process` JSON pipeline applies. See the official [Alloy install docs](https://grafana.com/docs/alloy/latest/set-up/install/kubernetes/) for the helm chart.
|
|
469
|
+
|
|
470
|
+
---
|
|
471
|
+
|
|
472
|
+
## Public API
|
|
473
|
+
|
|
474
|
+
Five symbols, no setup wrapper. The library provides only the things you can't get from stdlib alone; the `Logger` / `Handler` wiring is yours.
|
|
475
|
+
|
|
476
|
+
```python
|
|
477
|
+
from task_logging import (
|
|
478
|
+
TaskLogFilter, # logging.Filter — attach to a HANDLER (not a logger)
|
|
479
|
+
JsonFormatter, # logging.Formatter — emits one JSON line per record
|
|
480
|
+
task_log_context, # `with task_log_context({...}): ...`,
|
|
481
|
+
# or imperative ctx.enter() / ctx.exit()
|
|
482
|
+
get_task_log_attrs, # read the currently-active attrs (merged)
|
|
483
|
+
log_func_call, # decorator: ENTER / EXIT / RAISE for a function
|
|
484
|
+
)
|
|
485
|
+
```
|
|
486
|
+
|
|
487
|
+
| Symbol | Purpose |
|
|
488
|
+
|---|---|
|
|
489
|
+
| `TaskLogFilter(global_log_attrs=None)` | A `logging.Filter` that copies each record and merges `global_log_attrs` + the active `task_log_context` attrs onto it. **Attach to a handler, not a logger.** |
|
|
490
|
+
| `JsonFormatter(capture_locals=True)` | A `logging.Formatter` that emits one JSON line per record, with keys mirroring stdlib `LogRecord` attribute names. `capture_locals=False` disables the locals snapshot in exception logs. |
|
|
491
|
+
| `task_log_context(attrs)` | Bind a dict of attrs to the current execution context. Use `with task_log_context({...}):`, or imperative `ctx.enter()` / `ctx.exit()` for middleware that splits enter/exit across hooks. |
|
|
492
|
+
| `get_task_log_attrs()` | Return the currently-active merged attrs (empty dict if no context is active). |
|
|
493
|
+
| `log_func_call(logger=None, *, level=logging.INFO)` | Decorator that logs `ENTER` / `EXIT` / `RAISE` for a function. `logger=None` auto-resolves to `logging.getLogger(func.__module__)`. |
|
|
494
|
+
|
|
495
|
+
---
|
|
496
|
+
|
|
497
|
+
## End-to-end example
|
|
498
|
+
|
|
499
|
+
```python
|
|
500
|
+
import logging
|
|
501
|
+
import sys
|
|
502
|
+
from task_logging import (
|
|
503
|
+
JsonFormatter,
|
|
504
|
+
TaskLogFilter,
|
|
505
|
+
log_func_call,
|
|
506
|
+
task_log_context,
|
|
507
|
+
)
|
|
508
|
+
|
|
509
|
+
# Wire up stdlib once at startup.
|
|
510
|
+
handler = logging.StreamHandler(sys.stdout)
|
|
511
|
+
handler.setFormatter(JsonFormatter())
|
|
512
|
+
handler.addFilter(TaskLogFilter(global_log_attrs={"service": "Billing", "env": "prod"}))
|
|
513
|
+
root = logging.getLogger()
|
|
514
|
+
root.addHandler(handler)
|
|
515
|
+
root.setLevel(logging.INFO)
|
|
516
|
+
|
|
517
|
+
# Silence noisy third-party libraries via stdlib:
|
|
518
|
+
for chatty in ("urllib3", "botocore"):
|
|
519
|
+
logging.getLogger(chatty).setLevel(logging.WARNING)
|
|
520
|
+
|
|
521
|
+
log = logging.getLogger(__name__)
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
@log_func_call(log)
|
|
525
|
+
def charge(amount: float, currency: str) -> str:
|
|
526
|
+
return f"charged {amount} {currency}"
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
class Settlement:
|
|
530
|
+
@log_func_call(log, level=logging.DEBUG)
|
|
531
|
+
def settle(self, account: str) -> None:
|
|
532
|
+
log.info("settling %s", account)
|
|
533
|
+
try:
|
|
534
|
+
1 / 0
|
|
535
|
+
except ZeroDivisionError:
|
|
536
|
+
log.exception("settlement failed")
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def handle_request(req):
|
|
540
|
+
with task_log_context({"task_id": req.id, "user_id": req.user_id}):
|
|
541
|
+
charge(9.99, "USD")
|
|
542
|
+
Settlement().settle("acct-1")
|
|
543
|
+
```
|
|
544
|
+
|
|
545
|
+
When this runs in a container, every line of stdout is JSON tagged with the request's `task_id` and `user_id` — including any logs from `requests`, `urllib3`, `botocore`, etc. that fired during the request. Alloy picks the lines up via the Docker socket, attaches the `service=Billing` / `env=prod` labels from the container's labels, and ships them to Loki.
|
|
546
|
+
|
|
547
|
+
---
|
|
548
|
+
|
|
549
|
+
## Tips & gotchas
|
|
550
|
+
|
|
551
|
+
- **`service` must be low-cardinality.** It becomes a Loki label. Use `"OrderService"`, never `"OrderService-pod-abc-7"`.
|
|
552
|
+
- **`task_id` is per-request, never a label.** It rides inside the JSON payload. Loki ≥ 2.9 + `stage.structured_metadata` lets you filter on it efficiently.
|
|
553
|
+
- **Exception capture only works inside `except` blocks.** The formatter reads `sys.exc_info()`, so call `log.exception(...)` while the exception is still being handled.
|
|
554
|
+
- **Tests / hot-reloads need explicit teardown.** `addHandler` is additive — running your wiring twice stacks two handlers and logs each line twice. Stash the handler and remove it on teardown:
|
|
555
|
+
|
|
556
|
+
```python
|
|
557
|
+
def setup_for_tests():
|
|
558
|
+
handler = logging.StreamHandler(sys.stdout)
|
|
559
|
+
handler.setFormatter(JsonFormatter())
|
|
560
|
+
handler.addFilter(TaskLogFilter(...))
|
|
561
|
+
logging.getLogger().addHandler(handler)
|
|
562
|
+
return handler
|
|
563
|
+
|
|
564
|
+
def teardown_for_tests(handler):
|
|
565
|
+
logging.getLogger().removeHandler(handler)
|
|
566
|
+
handler.close()
|
|
567
|
+
```
|
|
568
|
+
|
|
569
|
+
- **Disable locals capture in regulated environments.** Pass `capture_locals=False` to `JsonFormatter()` if `repr()` of arbitrary local variables could leak secrets.
|
|
570
|
+
- **What about loguru?** loguru is not based on stdlib `logging`, so libraries like `requests` and `urllib3` won't be captured by it. This package deliberately uses stdlib so third-party logs flow through the same pipeline. If you want loguru in your own code, use loguru's `InterceptHandler` to bridge stdlib → loguru — but this library does not require it.
|
|
571
|
+
|
|
572
|
+
---
|
|
573
|
+
|
|
574
|
+
## Design notes
|
|
575
|
+
|
|
576
|
+
If you want a deeper mental model than this README provides, see [docs/](docs/):
|
|
577
|
+
|
|
578
|
+
- [`docs/design/decorators.md`](docs/design/decorators.md) — why one `@log_func_call` instead of `FunctionLogger` + `ClassFunctionLogger`, and why classes don't need a `_logger` attribute
|
|
579
|
+
- [`docs/design/task-context.md`](docs/design/task-context.md) — how `task_log_context` makes log attrs flow through threads, asyncio tasks, and third-party libraries' logs
|
|
580
|
+
- [`docs/design/stdlib-logging-primer.md`](docs/design/stdlib-logging-primer.md) — bottom-up tour of stdlib `logging` (LogRecord, the logger tree, handlers, filters, formatters) with the rules that prevent the most common pitfalls
|
|
581
|
+
- [`docs/design/why-json-logs.md`](docs/design/why-json-logs.md) — Loki accepts arbitrary text; why does this library emit JSON anyway? What do we gain, and what do we trade away?
|
|
582
|
+
- [`docs/design/json-schema.md`](docs/design/json-schema.md) — where the JSON keys come from, why we mirror stdlib LogRecord attribute names instead of inventing our own, and the stability promise
|
|
583
|
+
|
|
584
|
+
---
|
|
585
|
+
|
|
586
|
+
## Development
|
|
587
|
+
|
|
588
|
+
```bash
|
|
589
|
+
git clone https://github.com/im-zhong/task-logging.git
|
|
590
|
+
cd task-logging
|
|
591
|
+
uv sync
|
|
592
|
+
uv run pytest
|
|
593
|
+
```
|
|
594
|
+
|
|
595
|
+
```bash
|
|
596
|
+
uv run ruff check .
|
|
597
|
+
uv run mypy task_logging
|
|
598
|
+
uv run pre-commit run --all-files
|
|
599
|
+
```
|
|
600
|
+
|
|
601
|
+
---
|
|
602
|
+
|
|
603
|
+
## License
|
|
604
|
+
|
|
605
|
+
MIT — see [LICENSE](LICENSE).
|