boxfusion-log 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- boxfusion_log-0.1.0/.gitignore +14 -0
- boxfusion_log-0.1.0/CHANGELOG.md +16 -0
- boxfusion_log-0.1.0/LICENSE +21 -0
- boxfusion_log-0.1.0/PKG-INFO +252 -0
- boxfusion_log-0.1.0/README.md +214 -0
- boxfusion_log-0.1.0/pyproject.toml +58 -0
- boxfusion_log-0.1.0/src/boxlog/__init__.py +69 -0
- boxfusion_log-0.1.0/src/boxlog/__main__.py +5 -0
- boxfusion_log-0.1.0/src/boxlog/_record.py +39 -0
- boxfusion_log-0.1.0/src/boxlog/backends/__init__.py +65 -0
- boxfusion_log-0.1.0/src/boxlog/backends/base.py +29 -0
- boxfusion_log-0.1.0/src/boxlog/backends/memory.py +32 -0
- boxfusion_log-0.1.0/src/boxlog/backends/redis.py +59 -0
- boxfusion_log-0.1.0/src/boxlog/backends/sqlite.py +175 -0
- boxfusion_log-0.1.0/src/boxlog/cli.py +81 -0
- boxfusion_log-0.1.0/src/boxlog/configure.py +159 -0
- boxfusion_log-0.1.0/src/boxlog/context.py +61 -0
- boxfusion_log-0.1.0/src/boxlog/formatter.py +94 -0
- boxfusion_log-0.1.0/src/boxlog/handler.py +140 -0
- boxfusion_log-0.1.0/src/boxlog/middleware.py +177 -0
- boxfusion_log-0.1.0/src/boxlog/py.typed +0 -0
- boxfusion_log-0.1.0/src/boxlog/query.py +150 -0
- boxfusion_log-0.1.0/src/boxlog/redaction.py +121 -0
- boxfusion_log-0.1.0/src/boxlog/timing.py +104 -0
- boxfusion_log-0.1.0/src/boxlog/viewer/__init__.py +245 -0
- boxfusion_log-0.1.0/src/boxlog/viewer/static/index.html +72 -0
- boxfusion_log-0.1.0/src/boxlog/viewer/static/logs.css +221 -0
- boxfusion_log-0.1.0/src/boxlog/viewer/static/logs.js +339 -0
- boxfusion_log-0.1.0/tests/conftest.py +115 -0
- boxfusion_log-0.1.0/tests/test_backends.py +209 -0
- boxfusion_log-0.1.0/tests/test_core.py +224 -0
- boxfusion_log-0.1.0/tests/test_setup.py +135 -0
- boxfusion_log-0.1.0/tests/test_web.py +218 -0
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.0
|
|
4
|
+
|
|
5
|
+
First release, extracted from the RSL service-automation app.
|
|
6
|
+
|
|
7
|
+
- One-call setup: `boxlog.setup(service=...)`, with JSON or text output to stdout.
|
|
8
|
+
- Correlation context via `bind(...)`, carried across `await` and into asyncio tasks.
|
|
9
|
+
- `log_step()`, which logs start, ok (with `duration_ms`) or failed (with traceback) for a unit of
|
|
10
|
+
work. Each traceback is logged once, even when steps are nested.
|
|
11
|
+
- Secret redaction for bearer and basic auth, credentials embedded in URLs, and password, token,
|
|
12
|
+
key and cookie fields. You can add your own keys and patterns.
|
|
13
|
+
- A non-blocking handler that ships records to a backend off-thread in batches.
|
|
14
|
+
- Memory, SQLite and Redis backends, selectable with `backend_from_url()`.
|
|
15
|
+
- ASGI and WSGI request-id middleware.
|
|
16
|
+
- A web viewer as a dependency-free ASGI app you can mount anywhere, plus `boxlog serve`.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Boxfusion
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: boxfusion-log
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Structured JSON logging with request tracing, secret redaction, pluggable storage and a built-in web log viewer.
|
|
5
|
+
Author: Boxfusion
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Keywords: asgi,fastapi,json,log-viewer,logging,structured-logging,tracing
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Framework :: FastAPI
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: System :: Logging
|
|
20
|
+
Classifier: Typing :: Typed
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Provides-Extra: all
|
|
23
|
+
Requires-Dist: redis>=4.5; extra == 'all'
|
|
24
|
+
Requires-Dist: uvicorn>=0.23; extra == 'all'
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
27
|
+
Requires-Dist: httpx>=0.27; extra == 'dev'
|
|
28
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
29
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
30
|
+
Requires-Dist: redis>=4.5; extra == 'dev'
|
|
31
|
+
Requires-Dist: twine>=5; extra == 'dev'
|
|
32
|
+
Requires-Dist: uvicorn>=0.23; extra == 'dev'
|
|
33
|
+
Provides-Extra: redis
|
|
34
|
+
Requires-Dist: redis>=4.5; extra == 'redis'
|
|
35
|
+
Provides-Extra: web
|
|
36
|
+
Requires-Dist: uvicorn>=0.23; extra == 'web'
|
|
37
|
+
Description-Content-Type: text/markdown
|
|
38
|
+
|
|
39
|
+
# boxlog
|
|
40
|
+
|
|
41
|
+
Structured JSON logging for Python, with request and job tracing, secret redaction,
|
|
42
|
+
pluggable storage and a built-in web log viewer.
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install boxfusion-log # core: stdlib only, no dependencies
|
|
46
|
+
pip install "boxfusion-log[redis]" # + Redis storage
|
|
47
|
+
pip install "boxfusion-log[web]" # + the standalone `boxlog serve` viewer
|
|
48
|
+
pip install "boxfusion-log[all]"
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
import boxlog # the package installs as boxfusion-log but imports as boxlog
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
boxlog builds on the standard `logging` module, so it needs no new logger API. Your existing
|
|
56
|
+
`logging.getLogger(__name__)` calls, and every third-party library's logs, flow through it
|
|
57
|
+
unchanged.
|
|
58
|
+
|
|
59
|
+
## Quick start
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
import logging
|
|
63
|
+
import boxlog
|
|
64
|
+
|
|
65
|
+
boxlog.setup(service="billing", backend="sqlite:///logs/billing.db")
|
|
66
|
+
log = logging.getLogger(__name__)
|
|
67
|
+
|
|
68
|
+
log.info("Service started", extra={"event": "app.startup", "version": "1.4.2"})
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Output on stdout, one JSON object per line:
|
|
72
|
+
|
|
73
|
+
```json
|
|
74
|
+
{"ts": "2026-09-29T09:16:21.180+00:00", "level": "INFO", "logger": "__main__",
|
|
75
|
+
"message": "Service started", "event": "app.startup", "service": "billing",
|
|
76
|
+
"extra": {"version": "1.4.2"}}
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Use `stdout="text"` for readable lines during development.
|
|
80
|
+
|
|
81
|
+
## Logging a unit of work: `log_step`
|
|
82
|
+
|
|
83
|
+
One `with` block logs the start of the work, then either its success (with duration and
|
|
84
|
+
any result fields) or its failure (with the traceback), and re-raises the error:
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
with boxlog.log_step(log, "payment.charge", customer="c-42", amount=1999) as step:
|
|
88
|
+
receipt = gateway.charge(...)
|
|
89
|
+
step["receipt_id"] = receipt.id
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
| outcome | event | level | extras |
|
|
93
|
+
|---------|-------------------------|-------|---------------------------------------------------------|
|
|
94
|
+
| start | `payment.charge.start` | DEBUG | `customer`, `amount` |
|
|
95
|
+
| success | `payment.charge.ok` | INFO | `customer`, `amount`, `receipt_id`, `duration_ms` |
|
|
96
|
+
| failure | `payment.charge.failed` | ERROR | the fields, `duration_ms`, `error_type` and a traceback |
|
|
97
|
+
|
|
98
|
+
It works around `await` too. Nested steps log each traceback only once, in the innermost
|
|
99
|
+
step; outer steps still log `.failed`, so you see the chain of work that broke without
|
|
100
|
+
reading the same traceback five times. In your own `except` blocks, check
|
|
101
|
+
`boxlog.already_logged(exc)` for the same effect.
|
|
102
|
+
|
|
103
|
+
## Tracing: `bind`
|
|
104
|
+
|
|
105
|
+
Fields you bind are added to every record logged inside the block, including records
|
|
106
|
+
from awaited code and from asyncio tasks started inside it:
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
with boxlog.bind(job_id=job.id, tenant="acme"):
|
|
110
|
+
process(job) # every record in here carries job_id and tenant
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Field names must be lower_snake_case. The web middleware binds `request_id` for you.
|
|
114
|
+
|
|
115
|
+
## Web frameworks
|
|
116
|
+
|
|
117
|
+
**FastAPI / Starlette (ASGI):**
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
import os
|
|
121
|
+
from fastapi import FastAPI
|
|
122
|
+
import boxlog
|
|
123
|
+
|
|
124
|
+
boxlog.setup(service="api", backend="redis://localhost:6379/0?key=api:logs")
|
|
125
|
+
|
|
126
|
+
app = FastAPI()
|
|
127
|
+
app.add_middleware(boxlog.RequestIdMiddleware, quiet_paths=("/health", "/logs"))
|
|
128
|
+
app.mount("/logs", boxlog.create_viewer(
|
|
129
|
+
boxlog.backend_from_url("redis://localhost:6379/0?key=api:logs"),
|
|
130
|
+
token=os.environ["LOGS_TOKEN"],
|
|
131
|
+
title="API logs",
|
|
132
|
+
))
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
**Flask / Django (WSGI):**
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
app.wsgi_app = boxlog.WSGIRequestIdMiddleware(app.wsgi_app) # Flask
|
|
139
|
+
application = boxlog.WSGIRequestIdMiddleware(get_wsgi_application()) # Django
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Then view logs with the standalone viewer (below).
|
|
143
|
+
|
|
144
|
+
The middleware:
|
|
145
|
+
- reuses an incoming `X-Request-ID` if it's well-formed, and otherwise generates one;
|
|
146
|
+
- echoes the id back on the response;
|
|
147
|
+
- logs one `http.request` record per request (method, path, status and duration);
|
|
148
|
+
- logs any unhandled exception with its traceback.
|
|
149
|
+
|
|
150
|
+
## The web viewer
|
|
151
|
+
|
|
152
|
+
The viewer has:
|
|
153
|
+
- a level filter (including "Failures only");
|
|
154
|
+
- service and source filters, populated from your data;
|
|
155
|
+
- an event-prefix filter and text search;
|
|
156
|
+
- click-to-trace on any `request_id` or `job_id`;
|
|
157
|
+
- live tail;
|
|
158
|
+
- an expandable view of every record and its traceback.
|
|
159
|
+
|
|
160
|
+
- **Mounted** inside an ASGI app: `create_viewer(backend, token=...)`, as above.
|
|
161
|
+
- **Standalone**, for any app, including non-web workers and scripts:
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
export BOXLOG_TOKEN="$(boxlog token)"
|
|
165
|
+
boxlog serve sqlite:///logs/billing.db # http://127.0.0.1:8765/
|
|
166
|
+
boxlog serve "redis://localhost:6379/0?key=api:logs" --port 9000 --title "API logs"
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
**Security.** Logs contain whatever your users send you, so treat the viewer like an admin
|
|
170
|
+
console:
|
|
171
|
+
- It requires a bearer token of **at least 32 characters**. Generate one with `boxlog token`.
|
|
172
|
+
- If the token is unset, the mounted viewer answers 404 everywhere.
|
|
173
|
+
- The page renders log content only as text, never as HTML, and ships under a strict
|
|
174
|
+
Content-Security-Policy with no inline script.
|
|
175
|
+
- Serve it over HTTPS. The standalone server binds to `127.0.0.1` unless you pass `--host`.
|
|
176
|
+
|
|
177
|
+
`create_viewer` also takes callables for `backend` and `token`. That's useful when they're
|
|
178
|
+
only known at startup (for example, inside a lifespan).
|
|
179
|
+
|
|
180
|
+
## Storage backends
|
|
181
|
+
|
|
182
|
+
| backend | URL | good for |
|
|
183
|
+
|-----------------|------------------------------------------------------|----------------------------------------------------------------------------|
|
|
184
|
+
| `MemoryBackend` | `memory://?max=5000` | tests and scripts; lost on restart |
|
|
185
|
+
| `SQLiteBackend` | `sqlite:///logs/app.db?max=100000` | one machine, no server; indexed queries; processes on one host can share the file |
|
|
186
|
+
| `RedisBackend` | `redis://host:6379/0?key=app:logs&max=10000` | several processes or machines; survives restarts |
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
boxlog.setup(service="worker", backend=boxlog.SQLiteBackend("logs/worker.db"))
|
|
190
|
+
# or attach later, once settings are known:
|
|
191
|
+
boxlog.attach_backend("redis://cache:6379/0?key=worker:logs")
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
Records reach the backend from a background thread, in batches, so logging never blocks
|
|
195
|
+
your code or an event loop. If the backend is down, records are dropped (with one warning
|
|
196
|
+
on stderr) and your app carries on; stdout logging is unaffected. Several services can
|
|
197
|
+
share one backend, and the viewer shows a service filter when it finds more than one.
|
|
198
|
+
|
|
199
|
+
A custom backend needs three methods, `append(entries)`, `query(query)` and `close()`. See
|
|
200
|
+
`boxlog.backends.base.LogBackend`.
|
|
201
|
+
|
|
202
|
+
## Redaction
|
|
203
|
+
|
|
204
|
+
Before any output, boxlog masks:
|
|
205
|
+
- values of sensitive keys in `extra=` and bound fields: `password`, `token`, `api_key`,
|
|
206
|
+
`secret`, `authorization`, `cookie`, … (as whole names or `_` suffixes, so `access_token`
|
|
207
|
+
is masked but `total_tokens` isn't);
|
|
208
|
+
- `Bearer …` and `Basic …` credentials in text;
|
|
209
|
+
- `password=…` and `"token": "…"` pairs in text;
|
|
210
|
+
- passwords inside URLs (`redis://user:secret@host`);
|
|
211
|
+
- all of the above in exception messages and tracebacks.
|
|
212
|
+
|
|
213
|
+
You can add your own keys and patterns:
|
|
214
|
+
|
|
215
|
+
```python
|
|
216
|
+
boxlog.setup(redact_keys=["national_id"], redact_patterns=[r"\b\d{13}\b"])
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
Redaction is a safety net. Don't log secrets or whole request bodies on purpose.
|
|
220
|
+
|
|
221
|
+
## `setup()` reference
|
|
222
|
+
|
|
223
|
+
```python
|
|
224
|
+
boxlog.setup(
|
|
225
|
+
service=None, # tagged on every record
|
|
226
|
+
level="INFO",
|
|
227
|
+
stdout="json", # "json", "text" or None
|
|
228
|
+
backend=None, # LogBackend or URL
|
|
229
|
+
redact_keys=(), redact_patterns=(),
|
|
230
|
+
promoted_fields=("request_id", "job_id", "source"), # extra= keys lifted to top level
|
|
231
|
+
quiet_loggers=("httpx", "httpcore", "urllib3", "openai", "botocore", "redis", "asyncio"),
|
|
232
|
+
capture_uvicorn=True, # route uvicorn's logs through boxlog, drop its access log
|
|
233
|
+
)
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
To quiet more libraries while keeping the defaults, pass
|
|
237
|
+
`quiet_loggers=(*boxlog.DEFAULT_QUIET_LOGGERS, "pinecone")`.
|
|
238
|
+
|
|
239
|
+
You can call it again to change settings. It only replaces the handlers it installed
|
|
240
|
+
itself. `boxlog.shutdown()` (also registered with `atexit`) flushes queued records.
|
|
241
|
+
|
|
242
|
+
## Development
|
|
243
|
+
|
|
244
|
+
```bash
|
|
245
|
+
python -m venv .venv && .venv/bin/pip install -e ".[dev]"
|
|
246
|
+
.venv/bin/python -m pytest -q
|
|
247
|
+
.venv/bin/python -m build && .venv/bin/python -m twine check dist/*
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
## License
|
|
251
|
+
|
|
252
|
+
MIT
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
# boxlog
|
|
2
|
+
|
|
3
|
+
Structured JSON logging for Python, with request and job tracing, secret redaction,
|
|
4
|
+
pluggable storage and a built-in web log viewer.
|
|
5
|
+
|
|
6
|
+
```bash
|
|
7
|
+
pip install boxfusion-log # core: stdlib only, no dependencies
|
|
8
|
+
pip install "boxfusion-log[redis]" # + Redis storage
|
|
9
|
+
pip install "boxfusion-log[web]" # + the standalone `boxlog serve` viewer
|
|
10
|
+
pip install "boxfusion-log[all]"
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
```python
|
|
14
|
+
import boxlog # the package installs as boxfusion-log but imports as boxlog
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
boxlog builds on the standard `logging` module, so it needs no new logger API. Your existing
|
|
18
|
+
`logging.getLogger(__name__)` calls, and every third-party library's logs, flow through it
|
|
19
|
+
unchanged.
|
|
20
|
+
|
|
21
|
+
## Quick start
|
|
22
|
+
|
|
23
|
+
```python
|
|
24
|
+
import logging
|
|
25
|
+
import boxlog
|
|
26
|
+
|
|
27
|
+
boxlog.setup(service="billing", backend="sqlite:///logs/billing.db")
|
|
28
|
+
log = logging.getLogger(__name__)
|
|
29
|
+
|
|
30
|
+
log.info("Service started", extra={"event": "app.startup", "version": "1.4.2"})
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Output on stdout, one JSON object per line:
|
|
34
|
+
|
|
35
|
+
```json
|
|
36
|
+
{"ts": "2026-09-29T09:16:21.180+00:00", "level": "INFO", "logger": "__main__",
|
|
37
|
+
"message": "Service started", "event": "app.startup", "service": "billing",
|
|
38
|
+
"extra": {"version": "1.4.2"}}
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Use `stdout="text"` for readable lines during development.
|
|
42
|
+
|
|
43
|
+
## Logging a unit of work: `log_step`
|
|
44
|
+
|
|
45
|
+
One `with` block logs the start of the work, then either its success (with duration and
|
|
46
|
+
any result fields) or its failure (with the traceback), and re-raises the error:
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
with boxlog.log_step(log, "payment.charge", customer="c-42", amount=1999) as step:
|
|
50
|
+
receipt = gateway.charge(...)
|
|
51
|
+
step["receipt_id"] = receipt.id
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
| outcome | event | level | extras |
|
|
55
|
+
|---------|-------------------------|-------|---------------------------------------------------------|
|
|
56
|
+
| start | `payment.charge.start` | DEBUG | `customer`, `amount` |
|
|
57
|
+
| success | `payment.charge.ok` | INFO | `customer`, `amount`, `receipt_id`, `duration_ms` |
|
|
58
|
+
| failure | `payment.charge.failed` | ERROR | the fields, `duration_ms`, `error_type` and a traceback |
|
|
59
|
+
|
|
60
|
+
It works around `await` too. Nested steps log each traceback only once, in the innermost
|
|
61
|
+
step; outer steps still log `.failed`, so you see the chain of work that broke without
|
|
62
|
+
reading the same traceback five times. In your own `except` blocks, check
|
|
63
|
+
`boxlog.already_logged(exc)` for the same effect.
|
|
64
|
+
|
|
65
|
+
## Tracing: `bind`
|
|
66
|
+
|
|
67
|
+
Fields you bind are added to every record logged inside the block, including records
|
|
68
|
+
from awaited code and from asyncio tasks started inside it:
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
with boxlog.bind(job_id=job.id, tenant="acme"):
|
|
72
|
+
process(job) # every record in here carries job_id and tenant
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Field names must be lower_snake_case. The web middleware binds `request_id` for you.
|
|
76
|
+
|
|
77
|
+
## Web frameworks
|
|
78
|
+
|
|
79
|
+
**FastAPI / Starlette (ASGI):**
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
import os
|
|
83
|
+
from fastapi import FastAPI
|
|
84
|
+
import boxlog
|
|
85
|
+
|
|
86
|
+
boxlog.setup(service="api", backend="redis://localhost:6379/0?key=api:logs")
|
|
87
|
+
|
|
88
|
+
app = FastAPI()
|
|
89
|
+
app.add_middleware(boxlog.RequestIdMiddleware, quiet_paths=("/health", "/logs"))
|
|
90
|
+
app.mount("/logs", boxlog.create_viewer(
|
|
91
|
+
boxlog.backend_from_url("redis://localhost:6379/0?key=api:logs"),
|
|
92
|
+
token=os.environ["LOGS_TOKEN"],
|
|
93
|
+
title="API logs",
|
|
94
|
+
))
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
**Flask / Django (WSGI):**
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
app.wsgi_app = boxlog.WSGIRequestIdMiddleware(app.wsgi_app) # Flask
|
|
101
|
+
application = boxlog.WSGIRequestIdMiddleware(get_wsgi_application()) # Django
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Then view logs with the standalone viewer (below).
|
|
105
|
+
|
|
106
|
+
The middleware:
|
|
107
|
+
- reuses an incoming `X-Request-ID` if it's well-formed, and otherwise generates one;
|
|
108
|
+
- echoes the id back on the response;
|
|
109
|
+
- logs one `http.request` record per request (method, path, status and duration);
|
|
110
|
+
- logs any unhandled exception with its traceback.
|
|
111
|
+
|
|
112
|
+
## The web viewer
|
|
113
|
+
|
|
114
|
+
The viewer has:
|
|
115
|
+
- a level filter (including "Failures only");
|
|
116
|
+
- service and source filters, populated from your data;
|
|
117
|
+
- an event-prefix filter and text search;
|
|
118
|
+
- click-to-trace on any `request_id` or `job_id`;
|
|
119
|
+
- live tail;
|
|
120
|
+
- an expandable view of every record and its traceback.
|
|
121
|
+
|
|
122
|
+
- **Mounted** inside an ASGI app: `create_viewer(backend, token=...)`, as above.
|
|
123
|
+
- **Standalone**, for any app, including non-web workers and scripts:
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
export BOXLOG_TOKEN="$(boxlog token)"
|
|
127
|
+
boxlog serve sqlite:///logs/billing.db # http://127.0.0.1:8765/
|
|
128
|
+
boxlog serve "redis://localhost:6379/0?key=api:logs" --port 9000 --title "API logs"
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
**Security.** Logs contain whatever your users send you, so treat the viewer like an admin
|
|
132
|
+
console:
|
|
133
|
+
- It requires a bearer token of **at least 32 characters**. Generate one with `boxlog token`.
|
|
134
|
+
- If the token is unset, the mounted viewer answers 404 everywhere.
|
|
135
|
+
- The page renders log content only as text, never as HTML, and ships under a strict
|
|
136
|
+
Content-Security-Policy with no inline script.
|
|
137
|
+
- Serve it over HTTPS. The standalone server binds to `127.0.0.1` unless you pass `--host`.
|
|
138
|
+
|
|
139
|
+
`create_viewer` also takes callables for `backend` and `token`. That's useful when they're
|
|
140
|
+
only known at startup (for example, inside a lifespan).
|
|
141
|
+
|
|
142
|
+
## Storage backends
|
|
143
|
+
|
|
144
|
+
| backend | URL | good for |
|
|
145
|
+
|-----------------|------------------------------------------------------|----------------------------------------------------------------------------|
|
|
146
|
+
| `MemoryBackend` | `memory://?max=5000` | tests and scripts; lost on restart |
|
|
147
|
+
| `SQLiteBackend` | `sqlite:///logs/app.db?max=100000` | one machine, no server; indexed queries; processes on one host can share the file |
|
|
148
|
+
| `RedisBackend` | `redis://host:6379/0?key=app:logs&max=10000` | several processes or machines; survives restarts |
|
|
149
|
+
|
|
150
|
+
```python
|
|
151
|
+
boxlog.setup(service="worker", backend=boxlog.SQLiteBackend("logs/worker.db"))
|
|
152
|
+
# or attach later, once settings are known:
|
|
153
|
+
boxlog.attach_backend("redis://cache:6379/0?key=worker:logs")
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Records reach the backend from a background thread, in batches, so logging never blocks
|
|
157
|
+
your code or an event loop. If the backend is down, records are dropped (with one warning
|
|
158
|
+
on stderr) and your app carries on; stdout logging is unaffected. Several services can
|
|
159
|
+
share one backend, and the viewer shows a service filter when it finds more than one.
|
|
160
|
+
|
|
161
|
+
A custom backend needs three methods, `append(entries)`, `query(query)` and `close()`. See
|
|
162
|
+
`boxlog.backends.base.LogBackend`.
|
|
163
|
+
|
|
164
|
+
## Redaction
|
|
165
|
+
|
|
166
|
+
Before any output, boxlog masks:
|
|
167
|
+
- values of sensitive keys in `extra=` and bound fields: `password`, `token`, `api_key`,
|
|
168
|
+
`secret`, `authorization`, `cookie`, … (as whole names or `_` suffixes, so `access_token`
|
|
169
|
+
is masked but `total_tokens` isn't);
|
|
170
|
+
- `Bearer …` and `Basic …` credentials in text;
|
|
171
|
+
- `password=…` and `"token": "…"` pairs in text;
|
|
172
|
+
- passwords inside URLs (`redis://user:secret@host`);
|
|
173
|
+
- all of the above in exception messages and tracebacks.
|
|
174
|
+
|
|
175
|
+
You can add your own keys and patterns:
|
|
176
|
+
|
|
177
|
+
```python
|
|
178
|
+
boxlog.setup(redact_keys=["national_id"], redact_patterns=[r"\b\d{13}\b"])
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
Redaction is a safety net. Don't log secrets or whole request bodies on purpose.
|
|
182
|
+
|
|
183
|
+
## `setup()` reference
|
|
184
|
+
|
|
185
|
+
```python
|
|
186
|
+
boxlog.setup(
|
|
187
|
+
service=None, # tagged on every record
|
|
188
|
+
level="INFO",
|
|
189
|
+
stdout="json", # "json", "text" or None
|
|
190
|
+
backend=None, # LogBackend or URL
|
|
191
|
+
redact_keys=(), redact_patterns=(),
|
|
192
|
+
promoted_fields=("request_id", "job_id", "source"), # extra= keys lifted to top level
|
|
193
|
+
quiet_loggers=("httpx", "httpcore", "urllib3", "openai", "botocore", "redis", "asyncio"),
|
|
194
|
+
capture_uvicorn=True, # route uvicorn's logs through boxlog, drop its access log
|
|
195
|
+
)
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
To quiet more libraries while keeping the defaults, pass
|
|
199
|
+
`quiet_loggers=(*boxlog.DEFAULT_QUIET_LOGGERS, "pinecone")`.
|
|
200
|
+
|
|
201
|
+
You can call it again to change settings. It only replaces the handlers it installed
|
|
202
|
+
itself. `boxlog.shutdown()` (also registered with `atexit`) flushes queued records.
|
|
203
|
+
|
|
204
|
+
## Development
|
|
205
|
+
|
|
206
|
+
```bash
|
|
207
|
+
python -m venv .venv && .venv/bin/pip install -e ".[dev]"
|
|
208
|
+
.venv/bin/python -m pytest -q
|
|
209
|
+
.venv/bin/python -m build && .venv/bin/python -m twine check dist/*
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
## License
|
|
213
|
+
|
|
214
|
+
MIT
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.24"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "boxfusion-log"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Structured JSON logging with request tracing, secret redaction, pluggable storage and a built-in web log viewer."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Boxfusion" }]
|
|
14
|
+
keywords = ["logging", "structured-logging", "json", "tracing", "log-viewer", "asgi", "fastapi"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
21
|
+
"Programming Language :: Python :: 3.10",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Programming Language :: Python :: 3.13",
|
|
25
|
+
"Topic :: System :: Logging",
|
|
26
|
+
"Framework :: FastAPI",
|
|
27
|
+
"Typing :: Typed",
|
|
28
|
+
]
|
|
29
|
+
# The core is stdlib-only. Storage and the standalone viewer are opt-in.
|
|
30
|
+
dependencies = []
|
|
31
|
+
|
|
32
|
+
[project.optional-dependencies]
|
|
33
|
+
redis = ["redis>=4.5"]
|
|
34
|
+
web = ["uvicorn>=0.23"]
|
|
35
|
+
all = ["redis>=4.5", "uvicorn>=0.23"]
|
|
36
|
+
dev = [
|
|
37
|
+
"pytest>=8",
|
|
38
|
+
"pytest-asyncio>=0.23",
|
|
39
|
+
"httpx>=0.27",
|
|
40
|
+
"redis>=4.5",
|
|
41
|
+
"uvicorn>=0.23",
|
|
42
|
+
"build>=1.2",
|
|
43
|
+
"twine>=5",
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
[project.scripts]
|
|
47
|
+
boxlog = "boxlog.cli:main"
|
|
48
|
+
|
|
49
|
+
[tool.hatch.build.targets.wheel]
|
|
50
|
+
packages = ["src/boxlog"]
|
|
51
|
+
|
|
52
|
+
[tool.hatch.build.targets.sdist]
|
|
53
|
+
include = ["src/boxlog", "tests", "README.md", "LICENSE", "CHANGELOG.md"]
|
|
54
|
+
|
|
55
|
+
[tool.pytest.ini_options]
|
|
56
|
+
testpaths = ["tests"]
|
|
57
|
+
asyncio_mode = "auto"
|
|
58
|
+
filterwarnings = ["error::DeprecationWarning:boxlog.*"]
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""boxlog - structured JSON logging with request tracing, secret redaction, pluggable
|
|
2
|
+
storage and a built-in web log viewer.
|
|
3
|
+
|
|
4
|
+
import logging, boxlog
|
|
5
|
+
|
|
6
|
+
boxlog.setup(service="billing", backend="sqlite:///logs/billing.db")
|
|
7
|
+
log = logging.getLogger(__name__)
|
|
8
|
+
|
|
9
|
+
with boxlog.bind(request_id="abc"):
|
|
10
|
+
with boxlog.log_step(log, "invoice.create", customer="c-1") as step:
|
|
11
|
+
step["invoice_id"] = create_invoice()
|
|
12
|
+
|
|
13
|
+
It builds on the standard `logging` module: existing `logging.getLogger(...)` calls,
|
|
14
|
+
and third-party libraries' logs, all flow through it unchanged.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from boxlog.backends import (
|
|
18
|
+
LogBackend,
|
|
19
|
+
MemoryBackend,
|
|
20
|
+
RedisBackend,
|
|
21
|
+
SQLiteBackend,
|
|
22
|
+
backend_from_url,
|
|
23
|
+
)
|
|
24
|
+
from boxlog.configure import (
|
|
25
|
+
DEFAULT_QUIET_LOGGERS,
|
|
26
|
+
attach_backend,
|
|
27
|
+
detach_backend,
|
|
28
|
+
set_level,
|
|
29
|
+
setup,
|
|
30
|
+
shutdown,
|
|
31
|
+
)
|
|
32
|
+
from boxlog.context import bind, current
|
|
33
|
+
from boxlog.formatter import JsonFormatter, TextFormatter
|
|
34
|
+
from boxlog.handler import BackendHandler
|
|
35
|
+
from boxlog.middleware import RequestIdMiddleware, WSGIRequestIdMiddleware
|
|
36
|
+
from boxlog.query import LogQuery
|
|
37
|
+
from boxlog.redaction import Redactor
|
|
38
|
+
from boxlog.timing import already_logged, log_step, mark_logged
|
|
39
|
+
from boxlog.viewer import create_viewer
|
|
40
|
+
|
|
41
|
+
__version__ = "0.1.0"
|
|
42
|
+
|
|
43
|
+
__all__ = [
|
|
44
|
+
"DEFAULT_QUIET_LOGGERS",
|
|
45
|
+
"BackendHandler",
|
|
46
|
+
"JsonFormatter",
|
|
47
|
+
"LogBackend",
|
|
48
|
+
"LogQuery",
|
|
49
|
+
"MemoryBackend",
|
|
50
|
+
"Redactor",
|
|
51
|
+
"RedisBackend",
|
|
52
|
+
"RequestIdMiddleware",
|
|
53
|
+
"SQLiteBackend",
|
|
54
|
+
"TextFormatter",
|
|
55
|
+
"WSGIRequestIdMiddleware",
|
|
56
|
+
"__version__",
|
|
57
|
+
"already_logged",
|
|
58
|
+
"attach_backend",
|
|
59
|
+
"backend_from_url",
|
|
60
|
+
"bind",
|
|
61
|
+
"create_viewer",
|
|
62
|
+
"current",
|
|
63
|
+
"detach_backend",
|
|
64
|
+
"log_step",
|
|
65
|
+
"mark_logged",
|
|
66
|
+
"set_level",
|
|
67
|
+
"setup",
|
|
68
|
+
"shutdown",
|
|
69
|
+
]
|