boxfusion-log 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- boxfusion_log-0.1.0.dist-info/METADATA +252 -0
- boxfusion_log-0.1.0.dist-info/RECORD +27 -0
- boxfusion_log-0.1.0.dist-info/WHEEL +4 -0
- boxfusion_log-0.1.0.dist-info/entry_points.txt +2 -0
- boxfusion_log-0.1.0.dist-info/licenses/LICENSE +21 -0
- boxlog/__init__.py +69 -0
- boxlog/__main__.py +5 -0
- boxlog/_record.py +39 -0
- boxlog/backends/__init__.py +65 -0
- boxlog/backends/base.py +29 -0
- boxlog/backends/memory.py +32 -0
- boxlog/backends/redis.py +59 -0
- boxlog/backends/sqlite.py +175 -0
- boxlog/cli.py +81 -0
- boxlog/configure.py +159 -0
- boxlog/context.py +61 -0
- boxlog/formatter.py +94 -0
- boxlog/handler.py +140 -0
- boxlog/middleware.py +177 -0
- boxlog/py.typed +0 -0
- boxlog/query.py +150 -0
- boxlog/redaction.py +121 -0
- boxlog/timing.py +104 -0
- boxlog/viewer/__init__.py +245 -0
- boxlog/viewer/static/index.html +72 -0
- boxlog/viewer/static/logs.css +221 -0
- boxlog/viewer/static/logs.js +339 -0
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: boxfusion-log
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Structured JSON logging with request tracing, secret redaction, pluggable storage and a built-in web log viewer.
|
|
5
|
+
Author: Boxfusion
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Keywords: asgi,fastapi,json,log-viewer,logging,structured-logging,tracing
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Framework :: FastAPI
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: System :: Logging
|
|
20
|
+
Classifier: Typing :: Typed
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Provides-Extra: all
|
|
23
|
+
Requires-Dist: redis>=4.5; extra == 'all'
|
|
24
|
+
Requires-Dist: uvicorn>=0.23; extra == 'all'
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
27
|
+
Requires-Dist: httpx>=0.27; extra == 'dev'
|
|
28
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
29
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
30
|
+
Requires-Dist: redis>=4.5; extra == 'dev'
|
|
31
|
+
Requires-Dist: twine>=5; extra == 'dev'
|
|
32
|
+
Requires-Dist: uvicorn>=0.23; extra == 'dev'
|
|
33
|
+
Provides-Extra: redis
|
|
34
|
+
Requires-Dist: redis>=4.5; extra == 'redis'
|
|
35
|
+
Provides-Extra: web
|
|
36
|
+
Requires-Dist: uvicorn>=0.23; extra == 'web'
|
|
37
|
+
Description-Content-Type: text/markdown
|
|
38
|
+
|
|
39
|
+
# boxlog
|
|
40
|
+
|
|
41
|
+
Structured JSON logging for Python, with request and job tracing, secret redaction,
|
|
42
|
+
pluggable storage and a built-in web log viewer.
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install boxfusion-log # core: stdlib only, no dependencies
|
|
46
|
+
pip install "boxfusion-log[redis]" # + Redis storage
|
|
47
|
+
pip install "boxfusion-log[web]" # + the standalone `boxlog serve` viewer
|
|
48
|
+
pip install "boxfusion-log[all]"
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
import boxlog # the package installs as boxfusion-log but imports as boxlog
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
boxlog builds on the standard `logging` module, so it needs no new logger API. Your existing
|
|
56
|
+
`logging.getLogger(__name__)` calls, and every third-party library's logs, flow through it
|
|
57
|
+
unchanged.
|
|
58
|
+
|
|
59
|
+
## Quick start
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
import logging
|
|
63
|
+
import boxlog
|
|
64
|
+
|
|
65
|
+
boxlog.setup(service="billing", backend="sqlite:///logs/billing.db")
|
|
66
|
+
log = logging.getLogger(__name__)
|
|
67
|
+
|
|
68
|
+
log.info("Service started", extra={"event": "app.startup", "version": "1.4.2"})
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Output on stdout, one JSON object per line:
|
|
72
|
+
|
|
73
|
+
```json
|
|
74
|
+
{"ts": "2026-09-29T09:16:21.180+00:00", "level": "INFO", "logger": "__main__",
|
|
75
|
+
"message": "Service started", "event": "app.startup", "service": "billing",
|
|
76
|
+
"extra": {"version": "1.4.2"}}
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Use `stdout="text"` for readable lines during development.
|
|
80
|
+
|
|
81
|
+
## Logging a unit of work: `log_step`
|
|
82
|
+
|
|
83
|
+
One `with` block logs the start of the work, then either its success (with duration and
|
|
84
|
+
any result fields) or its failure (with the traceback), and re-raises the error:
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
with boxlog.log_step(log, "payment.charge", customer="c-42", amount=1999) as step:
|
|
88
|
+
receipt = gateway.charge(...)
|
|
89
|
+
step["receipt_id"] = receipt.id
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
| outcome | event | level | extras |
|
|
93
|
+
|---------|-------------------------|-------|---------------------------------------------------------|
|
|
94
|
+
| start | `payment.charge.start` | DEBUG | `customer`, `amount` |
|
|
95
|
+
| success | `payment.charge.ok` | INFO | `customer`, `amount`, `receipt_id`, `duration_ms` |
|
|
96
|
+
| failure | `payment.charge.failed` | ERROR | the fields, `duration_ms`, `error_type` and a traceback |
|
|
97
|
+
|
|
98
|
+
It works around `await` too. Nested steps log each traceback only once, in the innermost
|
|
99
|
+
step; outer steps still log `.failed`, so you see the chain of work that broke without
|
|
100
|
+
reading the same traceback five times. In your own `except` blocks, check
|
|
101
|
+
`boxlog.already_logged(exc)` for the same effect.
|
|
102
|
+
|
|
103
|
+
## Tracing: `bind`
|
|
104
|
+
|
|
105
|
+
Fields you bind are added to every record logged inside the block, including records
|
|
106
|
+
from awaited code and from asyncio tasks started inside it:
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
with boxlog.bind(job_id=job.id, tenant="acme"):
|
|
110
|
+
process(job) # every record in here carries job_id and tenant
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Field names must be lower_snake_case. The web middleware binds `request_id` for you.
|
|
114
|
+
|
|
115
|
+
## Web frameworks
|
|
116
|
+
|
|
117
|
+
**FastAPI / Starlette (ASGI):**
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
import os
|
|
121
|
+
from fastapi import FastAPI
|
|
122
|
+
import boxlog
|
|
123
|
+
|
|
124
|
+
boxlog.setup(service="api", backend="redis://localhost:6379/0?key=api:logs")
|
|
125
|
+
|
|
126
|
+
app = FastAPI()
|
|
127
|
+
app.add_middleware(boxlog.RequestIdMiddleware, quiet_paths=("/health", "/logs"))
|
|
128
|
+
app.mount("/logs", boxlog.create_viewer(
|
|
129
|
+
boxlog.backend_from_url("redis://localhost:6379/0?key=api:logs"),
|
|
130
|
+
token=os.environ["LOGS_TOKEN"],
|
|
131
|
+
title="API logs",
|
|
132
|
+
))
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
**Flask / Django (WSGI):**
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
app.wsgi_app = boxlog.WSGIRequestIdMiddleware(app.wsgi_app) # Flask
|
|
139
|
+
application = boxlog.WSGIRequestIdMiddleware(get_wsgi_application()) # Django
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Then view logs with the standalone viewer (below).
|
|
143
|
+
|
|
144
|
+
The middleware:
|
|
145
|
+
- reuses an incoming `X-Request-ID` if it's well-formed, and otherwise generates one;
|
|
146
|
+
- echoes the id back on the response;
|
|
147
|
+
- logs one `http.request` record per request (method, path, status and duration);
|
|
148
|
+
- logs any unhandled exception with its traceback.
|
|
149
|
+
|
|
150
|
+
## The web viewer
|
|
151
|
+
|
|
152
|
+
The viewer has:
|
|
153
|
+
- a level filter (including "Failures only");
|
|
154
|
+
- service and source filters, populated from your data;
|
|
155
|
+
- an event-prefix filter and text search;
|
|
156
|
+
- click-to-trace on any `request_id` or `job_id`;
|
|
157
|
+
- live tail;
|
|
158
|
+
- an expandable view of every record and its traceback.
|
|
159
|
+
|
|
160
|
+
- **Mounted** inside an ASGI app: `create_viewer(backend, token=...)`, as above.
|
|
161
|
+
- **Standalone**, for any app, including non-web workers and scripts:
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
export BOXLOG_TOKEN="$(boxlog token)"
|
|
165
|
+
boxlog serve sqlite:///logs/billing.db # http://127.0.0.1:8765/
|
|
166
|
+
boxlog serve "redis://localhost:6379/0?key=api:logs" --port 9000 --title "API logs"
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
**Security.** Logs contain whatever your users send you, so treat the viewer like an admin
|
|
170
|
+
console:
|
|
171
|
+
- It requires a bearer token of **at least 32 characters**. Generate one with `boxlog token`.
|
|
172
|
+
- If the token is unset, the mounted viewer answers 404 everywhere.
|
|
173
|
+
- The page renders log content only as text, never as HTML, and ships under a strict
|
|
174
|
+
Content-Security-Policy with no inline script.
|
|
175
|
+
- Serve it over HTTPS. The standalone server binds to `127.0.0.1` unless you pass `--host`.
|
|
176
|
+
|
|
177
|
+
`create_viewer` also takes callables for `backend` and `token`. That's useful when they're
|
|
178
|
+
only known at startup (for example, inside a lifespan).
|
|
179
|
+
|
|
180
|
+
## Storage backends
|
|
181
|
+
|
|
182
|
+
| backend | URL | good for |
|
|
183
|
+
|-----------------|------------------------------------------------------|----------------------------------------------------------------------------|
|
|
184
|
+
| `MemoryBackend` | `memory://?max=5000` | tests and scripts; lost on restart |
|
|
185
|
+
| `SQLiteBackend` | `sqlite:///logs/app.db?max=100000` | one machine, no server; indexed queries; processes on one host can share the file |
|
|
186
|
+
| `RedisBackend` | `redis://host:6379/0?key=app:logs&max=10000` | several processes or machines; survives restarts |
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
boxlog.setup(service="worker", backend=boxlog.SQLiteBackend("logs/worker.db"))
|
|
190
|
+
# or attach later, once settings are known:
|
|
191
|
+
boxlog.attach_backend("redis://cache:6379/0?key=worker:logs")
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
Records reach the backend from a background thread, in batches, so logging never blocks
|
|
195
|
+
your code or an event loop. If the backend is down, records are dropped (with one warning
|
|
196
|
+
on stderr) and your app carries on; stdout logging is unaffected. Several services can
|
|
197
|
+
share one backend, and the viewer shows a service filter when it finds more than one.
|
|
198
|
+
|
|
199
|
+
A custom backend needs three methods, `append(entries)`, `query(query)` and `close()`. See
|
|
200
|
+
`boxlog.backends.base.LogBackend`.
|
|
201
|
+
|
|
202
|
+
## Redaction
|
|
203
|
+
|
|
204
|
+
Before any output, boxlog masks:
|
|
205
|
+
- values of sensitive keys in `extra=` and bound fields: `password`, `token`, `api_key`,
|
|
206
|
+
`secret`, `authorization`, `cookie`, … (as whole names or `_` suffixes, so `access_token`
|
|
207
|
+
is masked but `total_tokens` isn't);
|
|
208
|
+
- `Bearer …` and `Basic …` credentials in text;
|
|
209
|
+
- `password=…` and `"token": "…"` pairs in text;
|
|
210
|
+
- passwords inside URLs (`redis://user:secret@host`);
|
|
211
|
+
- all of the above in exception messages and tracebacks.
|
|
212
|
+
|
|
213
|
+
You can add your own keys and patterns:
|
|
214
|
+
|
|
215
|
+
```python
|
|
216
|
+
boxlog.setup(redact_keys=["national_id"], redact_patterns=[r"\b\d{13}\b"])
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
Redaction is a safety net. Don't log secrets or whole request bodies on purpose.
|
|
220
|
+
|
|
221
|
+
## `setup()` reference
|
|
222
|
+
|
|
223
|
+
```python
|
|
224
|
+
boxlog.setup(
|
|
225
|
+
service=None, # tagged on every record
|
|
226
|
+
level="INFO",
|
|
227
|
+
stdout="json", # "json", "text" or None
|
|
228
|
+
backend=None, # LogBackend or URL
|
|
229
|
+
redact_keys=(), redact_patterns=(),
|
|
230
|
+
promoted_fields=("request_id", "job_id", "source"), # extra= keys lifted to top level
|
|
231
|
+
quiet_loggers=("httpx", "httpcore", "urllib3", "openai", "botocore", "redis", "asyncio"),
|
|
232
|
+
capture_uvicorn=True, # route uvicorn's logs through boxlog, drop its access log
|
|
233
|
+
)
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
To quiet more libraries while keeping the defaults, pass
|
|
237
|
+
`quiet_loggers=(*boxlog.DEFAULT_QUIET_LOGGERS, "pinecone")`.
|
|
238
|
+
|
|
239
|
+
You can call it again to change settings. It only replaces the handlers it installed
|
|
240
|
+
itself. `boxlog.shutdown()` (also registered with `atexit`) flushes queued records.
|
|
241
|
+
|
|
242
|
+
## Development
|
|
243
|
+
|
|
244
|
+
```bash
|
|
245
|
+
python -m venv .venv && .venv/bin/pip install -e ".[dev]"
|
|
246
|
+
.venv/bin/python -m pytest -q
|
|
247
|
+
.venv/bin/python -m build && .venv/bin/python -m twine check dist/*
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
## License
|
|
251
|
+
|
|
252
|
+
MIT
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
boxlog/__init__.py,sha256=JTFfGawbKXgZSR1zxMw41Vfz_fYYptJgmQmwXymCVgw,1785
|
|
2
|
+
boxlog/__main__.py,sha256=YwHfeAqB5e375f-y8_feGWX2W_l7TQdIfsHTzrAZFSM,58
|
|
3
|
+
boxlog/_record.py,sha256=i4OTvz2N6TAMEniStWMAK9PPHGsdapn5XfMhIruIm48,1272
|
|
4
|
+
boxlog/cli.py,sha256=w6ufISiw36YifzhMGklKlOAn_K3l2BokivVJ0iWTIuk,2850
|
|
5
|
+
boxlog/configure.py,sha256=0OHMrcLruRAy5AIviLFPp1-E-Ck98fwq-Ro-cpG1-b4,5997
|
|
6
|
+
boxlog/context.py,sha256=GvD9Vx1YIIndGq6JOXY4sgNgUnAELm0CTDWf3PdKdAs,1998
|
|
7
|
+
boxlog/formatter.py,sha256=bU6VzAvvgbXsiR1eZ2Pyk_d7Km1W_4P7SVQSTQfgoK4,3380
|
|
8
|
+
boxlog/handler.py,sha256=6FUgsoFwTuJpunvTVJbg8NWpGwl5uIIa1NBIVABPjUs,4733
|
|
9
|
+
boxlog/middleware.py,sha256=DjmtQ7i5PzJXDBA2H-IS_hCSod_7Ue_ZEAfCXUdNM2c,6139
|
|
10
|
+
boxlog/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
|
+
boxlog/query.py,sha256=o5GVzQSfoSFaYTwSFHhU2Ni6QSvR3KE5WkjDSMvHWCc,5555
|
|
12
|
+
boxlog/redaction.py,sha256=3Wa08PQbBXTfeKV2CvS4VHfyGXbSvgTDXdU1aMCxSik,4565
|
|
13
|
+
boxlog/timing.py,sha256=Wc709daB-DOxd3l7GdE_5FR7y-54Q4NrzVM27ooDKOY,3507
|
|
14
|
+
boxlog/backends/__init__.py,sha256=QZJvoUY0Q7am4rIveDK3ouh6bRE-OEenufw_HOJskeQ,2393
|
|
15
|
+
boxlog/backends/base.py,sha256=s0LD9rs5rQQ8JXYW9Sr32bkdsxuUCtARd5IramXM6p4,892
|
|
16
|
+
boxlog/backends/memory.py,sha256=IsxiAUExh1zFeFbklYq0DodnbBvf1dCSBcymtOlaKOs,906
|
|
17
|
+
boxlog/backends/redis.py,sha256=6Z9q5JK966evO71fx-oJEDDQzCaa7ex8lAR8jcnCKpI,1954
|
|
18
|
+
boxlog/backends/sqlite.py,sha256=K7J5eht4QxvoP0_58AzPX3rJI1G3cakNO4N_iMBsNC0,6284
|
|
19
|
+
boxlog/viewer/__init__.py,sha256=WStKSfwXb9OLS1mBKhBEQQu_-KZquggKnrmWaMAItXc,9186
|
|
20
|
+
boxlog/viewer/static/index.html,sha256=vOWLZuhHeqMoGxlF6FfZNWUxu-EcoXOMz9o3vrrSwts,2835
|
|
21
|
+
boxlog/viewer/static/logs.css,sha256=RPVBDetg_aHsGkgro7ZPuReurpIf5DtozkBxDVbUvVM,5799
|
|
22
|
+
boxlog/viewer/static/logs.js,sha256=hlgbDIsLHJObTMbH--8NTrGBbS-1XtIrDJIvJ4aPGbI,11380
|
|
23
|
+
boxfusion_log-0.1.0.dist-info/METADATA,sha256=zyt8qCs_1QH5w54_BUhr8S7CdL_FjpzSpU23bmg_Pf0,9830
|
|
24
|
+
boxfusion_log-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
25
|
+
boxfusion_log-0.1.0.dist-info/entry_points.txt,sha256=XZJ-vIgYibicvdhseh_fyevzfkEKufU80wVVL8j23ps,43
|
|
26
|
+
boxfusion_log-0.1.0.dist-info/licenses/LICENSE,sha256=Xz2lCy18SXQLSsNYV_oXO7wd-Pfe-HzxgSLlBKmICu0,1066
|
|
27
|
+
boxfusion_log-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Boxfusion
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
boxlog/__init__.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""boxlog - structured JSON logging with request tracing, secret redaction, pluggable
|
|
2
|
+
storage and a built-in web log viewer.
|
|
3
|
+
|
|
4
|
+
import logging, boxlog
|
|
5
|
+
|
|
6
|
+
boxlog.setup(service="billing", backend="sqlite:///logs/billing.db")
|
|
7
|
+
log = logging.getLogger(__name__)
|
|
8
|
+
|
|
9
|
+
with boxlog.bind(request_id="abc"):
|
|
10
|
+
with boxlog.log_step(log, "invoice.create", customer="c-1") as step:
|
|
11
|
+
step["invoice_id"] = create_invoice()
|
|
12
|
+
|
|
13
|
+
It builds on the standard `logging` module: existing `logging.getLogger(...)` calls,
|
|
14
|
+
and third-party libraries' logs, all flow through it unchanged.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from boxlog.backends import (
|
|
18
|
+
LogBackend,
|
|
19
|
+
MemoryBackend,
|
|
20
|
+
RedisBackend,
|
|
21
|
+
SQLiteBackend,
|
|
22
|
+
backend_from_url,
|
|
23
|
+
)
|
|
24
|
+
from boxlog.configure import (
|
|
25
|
+
DEFAULT_QUIET_LOGGERS,
|
|
26
|
+
attach_backend,
|
|
27
|
+
detach_backend,
|
|
28
|
+
set_level,
|
|
29
|
+
setup,
|
|
30
|
+
shutdown,
|
|
31
|
+
)
|
|
32
|
+
from boxlog.context import bind, current
|
|
33
|
+
from boxlog.formatter import JsonFormatter, TextFormatter
|
|
34
|
+
from boxlog.handler import BackendHandler
|
|
35
|
+
from boxlog.middleware import RequestIdMiddleware, WSGIRequestIdMiddleware
|
|
36
|
+
from boxlog.query import LogQuery
|
|
37
|
+
from boxlog.redaction import Redactor
|
|
38
|
+
from boxlog.timing import already_logged, log_step, mark_logged
|
|
39
|
+
from boxlog.viewer import create_viewer
|
|
40
|
+
|
|
41
|
+
__version__ = "0.1.0"
|
|
42
|
+
|
|
43
|
+
__all__ = [
|
|
44
|
+
"DEFAULT_QUIET_LOGGERS",
|
|
45
|
+
"BackendHandler",
|
|
46
|
+
"JsonFormatter",
|
|
47
|
+
"LogBackend",
|
|
48
|
+
"LogQuery",
|
|
49
|
+
"MemoryBackend",
|
|
50
|
+
"Redactor",
|
|
51
|
+
"RedisBackend",
|
|
52
|
+
"RequestIdMiddleware",
|
|
53
|
+
"SQLiteBackend",
|
|
54
|
+
"TextFormatter",
|
|
55
|
+
"WSGIRequestIdMiddleware",
|
|
56
|
+
"__version__",
|
|
57
|
+
"already_logged",
|
|
58
|
+
"attach_backend",
|
|
59
|
+
"backend_from_url",
|
|
60
|
+
"bind",
|
|
61
|
+
"create_viewer",
|
|
62
|
+
"current",
|
|
63
|
+
"detach_backend",
|
|
64
|
+
"log_step",
|
|
65
|
+
"mark_logged",
|
|
66
|
+
"set_level",
|
|
67
|
+
"setup",
|
|
68
|
+
"shutdown",
|
|
69
|
+
]
|
boxlog/__main__.py
ADDED
boxlog/_record.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Helpers for telling a LogRecord's own attributes apart from `extra=` fields."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
STANDARD_RECORD_ATTRS = frozenset(
|
|
9
|
+
vars(logging.LogRecord("", 0, "", 0, "", None, None)).keys()
|
|
10
|
+
) | {"message", "asctime", "taskName"}
|
|
11
|
+
|
|
12
|
+
# Attributes boxlog's own filters attach to a record.
|
|
13
|
+
CONTEXT_ATTR = "boxlog_context"
|
|
14
|
+
EXC_ATTR = "boxlog_exc"
|
|
15
|
+
INTERNAL_ATTRS = frozenset({CONTEXT_ATTR, EXC_ATTR})
|
|
16
|
+
|
|
17
|
+
# Extras other libraries attach that carry no information (uvicorn adds an ANSI-coloured
|
|
18
|
+
# copy of every message).
|
|
19
|
+
IGNORED_EXTRA_KEYS = frozenset({"color_message"})
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def extra_fields(record: logging.LogRecord) -> dict[str, Any]:
|
|
23
|
+
return {
|
|
24
|
+
key: value
|
|
25
|
+
for key, value in vars(record).items()
|
|
26
|
+
if key not in STANDARD_RECORD_ATTRS
|
|
27
|
+
and key not in INTERNAL_ATTRS
|
|
28
|
+
and key not in IGNORED_EXTRA_KEYS
|
|
29
|
+
and not key.startswith("_")
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def safe_extra(fields: dict[str, Any]) -> dict[str, Any]:
|
|
34
|
+
"""`extra=` keys that collide with LogRecord attributes make logging raise KeyError -
|
|
35
|
+
prefix them instead of crashing the caller."""
|
|
36
|
+
return {
|
|
37
|
+
(f"field_{key}" if key in STANDARD_RECORD_ATTRS else key): value
|
|
38
|
+
for key, value in fields.items()
|
|
39
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Where records are persisted for the viewer.
|
|
2
|
+
|
|
3
|
+
MemoryBackend in-process ring buffer (tests, scripts)
|
|
4
|
+
SQLiteBackend a local file, indexed queries (single machine, no server)
|
|
5
|
+
RedisBackend a capped Redis list (several processes / machines)
|
|
6
|
+
|
|
7
|
+
`backend_from_url()` builds one from a URL, which is handy for configuration:
|
|
8
|
+
memory:// memory://?max=5000
|
|
9
|
+
sqlite:///logs/app.db sqlite:///C:/logs/app.db sqlite:///:memory:
|
|
10
|
+
redis://host:6379/0?key=myapp:logs&max=20000 rediss://...
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from urllib.parse import parse_qs, urlsplit
|
|
16
|
+
|
|
17
|
+
from boxlog.backends.base import LogBackend, LogEntry
|
|
18
|
+
from boxlog.backends.memory import MemoryBackend
|
|
19
|
+
from boxlog.backends.redis import RedisBackend
|
|
20
|
+
from boxlog.backends.sqlite import SQLiteBackend
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"LogBackend",
|
|
24
|
+
"LogEntry",
|
|
25
|
+
"MemoryBackend",
|
|
26
|
+
"RedisBackend",
|
|
27
|
+
"SQLiteBackend",
|
|
28
|
+
"backend_from_url",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def backend_from_url(url: str) -> LogBackend:
|
|
33
|
+
parts = urlsplit(url)
|
|
34
|
+
scheme = parts.scheme.lower()
|
|
35
|
+
params = {k: v[-1] for k, v in parse_qs(parts.query).items()}
|
|
36
|
+
|
|
37
|
+
def max_records(default: int) -> int:
|
|
38
|
+
raw = params.get("max")
|
|
39
|
+
if raw is None:
|
|
40
|
+
return default
|
|
41
|
+
value = int(raw)
|
|
42
|
+
if value < 1:
|
|
43
|
+
raise ValueError("max must be positive")
|
|
44
|
+
return value
|
|
45
|
+
|
|
46
|
+
if scheme == "memory":
|
|
47
|
+
return MemoryBackend(max_records=max_records(10_000))
|
|
48
|
+
if scheme == "sqlite":
|
|
49
|
+
path = parts.path
|
|
50
|
+
# sqlite:///C:/x.db arrives as "/C:/x.db"; sqlite:///rel.db as "/rel.db".
|
|
51
|
+
if len(path) >= 3 and path[0] == "/" and path[2] == ":":
|
|
52
|
+
path = path[1:]
|
|
53
|
+
elif path.startswith("//"):
|
|
54
|
+
path = path[1:] # sqlite:////abs/path.db -> /abs/path.db
|
|
55
|
+
elif path.startswith("/"):
|
|
56
|
+
path = path[1:] # relative to the working directory
|
|
57
|
+
if not path:
|
|
58
|
+
raise ValueError("sqlite URL needs a path, e.g. sqlite:///logs.db")
|
|
59
|
+
return SQLiteBackend(path, max_records=max_records(100_000))
|
|
60
|
+
if scheme in ("redis", "rediss", "unix"):
|
|
61
|
+
clean = url.split("?", 1)[0]
|
|
62
|
+
return RedisBackend(
|
|
63
|
+
clean, key=params.get("key", "boxlog"), max_records=max_records(10_000)
|
|
64
|
+
)
|
|
65
|
+
raise ValueError(f"Unsupported backend URL scheme {scheme!r} (memory, sqlite, redis)")
|
boxlog/backends/base.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""The backend protocol.
|
|
2
|
+
|
|
3
|
+
Backends are synchronous. append() is only ever called from the handler's shipper
|
|
4
|
+
thread, and the viewer runs query() in a worker thread, so a backend never blocks an
|
|
5
|
+
event loop.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import Any, NamedTuple, Protocol, runtime_checkable
|
|
11
|
+
|
|
12
|
+
from boxlog.query import LogQuery
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class LogEntry(NamedTuple):
|
|
16
|
+
record: dict[str, Any] # the structured record (for backends that index columns)
|
|
17
|
+
line: str # the same record as a JSON line (for backends that store text)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@runtime_checkable
|
|
21
|
+
class LogBackend(Protocol):
|
|
22
|
+
def append(self, entries: list[LogEntry]) -> None: ...
|
|
23
|
+
|
|
24
|
+
def query(self, query: LogQuery) -> dict[str, Any]:
|
|
25
|
+
"""Return {"records": [...newest first], "latest": id|None, "count": n,
|
|
26
|
+
"facets": {"services": [...], "sources": [...]}}."""
|
|
27
|
+
...
|
|
28
|
+
|
|
29
|
+
def close(self) -> None: ...
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""In-process ring buffer. Zero setup, lost on restart - good for tests and scripts."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import threading
|
|
6
|
+
from collections import deque
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from boxlog.backends.base import LogEntry
|
|
10
|
+
from boxlog.query import LogQuery, filter_lines
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class MemoryBackend:
|
|
14
|
+
def __init__(self, max_records: int = 10_000) -> None:
|
|
15
|
+
self._lines: deque[str] = deque(maxlen=max_records)
|
|
16
|
+
self._lock = threading.Lock()
|
|
17
|
+
|
|
18
|
+
def append(self, entries: list[LogEntry]) -> None:
|
|
19
|
+
with self._lock:
|
|
20
|
+
for entry in entries:
|
|
21
|
+
self._lines.appendleft(entry.line)
|
|
22
|
+
|
|
23
|
+
def lines(self) -> list[str]:
|
|
24
|
+
"""Newest first."""
|
|
25
|
+
with self._lock:
|
|
26
|
+
return list(self._lines)
|
|
27
|
+
|
|
28
|
+
def query(self, query: LogQuery) -> dict[str, Any]:
|
|
29
|
+
return filter_lines(self.lines(), query)
|
|
30
|
+
|
|
31
|
+
def close(self) -> None:
|
|
32
|
+
pass
|
boxlog/backends/redis.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""A capped Redis list: LPUSH + LTRIM, newest first. Shared across processes and
|
|
2
|
+
machines, survives restarts. Needs `pip install boxfusion-log[redis]`."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from typing import Any, Optional
|
|
7
|
+
|
|
8
|
+
from boxlog.backends.base import LogEntry
|
|
9
|
+
from boxlog.query import LogQuery, filter_lines
|
|
10
|
+
|
|
11
|
+
DEFAULT_KEY = "boxlog"
|
|
12
|
+
DEFAULT_MAX_RECORDS = 10_000
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class RedisBackend:
|
|
16
|
+
def __init__(
|
|
17
|
+
self,
|
|
18
|
+
url: Optional[str] = None,
|
|
19
|
+
*,
|
|
20
|
+
client: Any = None,
|
|
21
|
+
key: str = DEFAULT_KEY,
|
|
22
|
+
max_records: int = DEFAULT_MAX_RECORDS,
|
|
23
|
+
) -> None:
|
|
24
|
+
if client is None:
|
|
25
|
+
if url is None:
|
|
26
|
+
raise ValueError("RedisBackend needs either url or client")
|
|
27
|
+
try:
|
|
28
|
+
from redis import Redis
|
|
29
|
+
except ImportError as exc: # pragma: no cover - depends on the environment
|
|
30
|
+
raise ImportError(
|
|
31
|
+
"RedisBackend needs the redis package: pip install 'boxfusion-log[redis]'"
|
|
32
|
+
) from exc
|
|
33
|
+
client = Redis.from_url(
|
|
34
|
+
url, decode_responses=True, socket_timeout=5, socket_connect_timeout=5
|
|
35
|
+
)
|
|
36
|
+
self._redis = client
|
|
37
|
+
self._key = key
|
|
38
|
+
self._max = max_records
|
|
39
|
+
|
|
40
|
+
def append(self, entries: list[LogEntry]) -> None:
|
|
41
|
+
if not entries:
|
|
42
|
+
return
|
|
43
|
+
pipe = self._redis.pipeline(transaction=False)
|
|
44
|
+
pipe.lpush(self._key, *(entry.line for entry in entries))
|
|
45
|
+
pipe.ltrim(self._key, 0, self._max - 1)
|
|
46
|
+
pipe.execute()
|
|
47
|
+
|
|
48
|
+
def query(self, query: LogQuery) -> dict[str, Any]:
|
|
49
|
+
lines = self._redis.lrange(self._key, 0, self._max - 1)
|
|
50
|
+
return filter_lines(
|
|
51
|
+
(line.decode("utf-8", "replace") if isinstance(line, bytes) else line for line in lines),
|
|
52
|
+
query,
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
def close(self) -> None:
|
|
56
|
+
try:
|
|
57
|
+
self._redis.close()
|
|
58
|
+
except Exception: # noqa: BLE001
|
|
59
|
+
pass
|