nilva-django-logger 0.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nilva_django_logger-0.1.4/LICENSE +21 -0
- nilva_django_logger-0.1.4/PKG-INFO +246 -0
- nilva_django_logger-0.1.4/README.md +223 -0
- nilva_django_logger-0.1.4/nilva_django_logger/__init__.py +59 -0
- nilva_django_logger-0.1.4/nilva_django_logger/banner.py +58 -0
- nilva_django_logger-0.1.4/nilva_django_logger/config.py +183 -0
- nilva_django_logger-0.1.4/nilva_django_logger/filters.py +237 -0
- nilva_django_logger-0.1.4/nilva_django_logger/formatters.py +303 -0
- nilva_django_logger-0.1.4/nilva_django_logger/health.py +63 -0
- nilva_django_logger-0.1.4/nilva_django_logger/middleware.py +264 -0
- nilva_django_logger-0.1.4/nilva_django_logger/records.py +80 -0
- nilva_django_logger-0.1.4/nilva_django_logger/settings.py +137 -0
- nilva_django_logger-0.1.4/nilva_django_logger.egg-info/PKG-INFO +246 -0
- nilva_django_logger-0.1.4/nilva_django_logger.egg-info/SOURCES.txt +30 -0
- nilva_django_logger-0.1.4/nilva_django_logger.egg-info/dependency_links.txt +1 -0
- nilva_django_logger-0.1.4/nilva_django_logger.egg-info/requires.txt +6 -0
- nilva_django_logger-0.1.4/nilva_django_logger.egg-info/top_level.txt +1 -0
- nilva_django_logger-0.1.4/pyproject.toml +61 -0
- nilva_django_logger-0.1.4/setup.cfg +4 -0
- nilva_django_logger-0.1.4/tests/test_audit_fixes.py +169 -0
- nilva_django_logger-0.1.4/tests/test_colorful_formatter.py +62 -0
- nilva_django_logger-0.1.4/tests/test_config.py +93 -0
- nilva_django_logger-0.1.4/tests/test_edge_cases.py +234 -0
- nilva_django_logger-0.1.4/tests/test_filters.py +124 -0
- nilva_django_logger-0.1.4/tests/test_full_flow.py +106 -0
- nilva_django_logger-0.1.4/tests/test_health.py +51 -0
- nilva_django_logger-0.1.4/tests/test_json_formatter.py +90 -0
- nilva_django_logger-0.1.4/tests/test_middleware.py +166 -0
- nilva_django_logger-0.1.4/tests/test_package.py +55 -0
- nilva_django_logger-0.1.4/tests/test_recheck_fixes.py +71 -0
- nilva_django_logger-0.1.4/tests/test_request_json_formatter.py +46 -0
- nilva_django_logger-0.1.4/tests/test_settings.py +53 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023 Nilva
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nilva-django-logger
|
|
3
|
+
Version: 0.1.4
|
|
4
|
+
Summary: Nilva house logging for Django and Python services — structured JSON logs, colourful console, secret masking, request logging, health probes
|
|
5
|
+
Author-email: Nilva <dev@nilva.ai>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: homepage, https://gitlab.nilva.ai/nilva/dev-assets/django/nilva-django-logger
|
|
8
|
+
Project-URL: repository, https://gitlab.nilva.ai/nilva/dev-assets/django/nilva-django-logger
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Framework :: Django
|
|
13
|
+
Classifier: Framework :: Django :: 6.0
|
|
14
|
+
Classifier: Topic :: System :: Logging
|
|
15
|
+
Requires-Python: >=3.14
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Provides-Extra: django
|
|
19
|
+
Requires-Dist: django>=6.0; extra == "django"
|
|
20
|
+
Provides-Extra: jalali
|
|
21
|
+
Requires-Dist: jdatetime>=4.1; extra == "jalali"
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# nilva-django-logger
|
|
25
|
+
|
|
26
|
+
The Nilva house logging toolkit — structured one-JSON-object-per-line logs
|
|
27
|
+
(shipped by Filebeat), a colourful self-contained console formatter, secret
|
|
28
|
+
masking, request/response logging middleware, Kubernetes health probes and
|
|
29
|
+
probe-noise filtering.
|
|
30
|
+
|
|
31
|
+
It consolidates the logging stack that grew across Nilva backends —
|
|
32
|
+
`shahab-backend` (`shahab/log_formatter.py`, `settings/logging.py`,
|
|
33
|
+
`settings/log_filter.py`, `health.py`, `banner.py`), `mymci-bubble-backend`
|
|
34
|
+
(`core/log_formatter.py`, `core/middlewares/logger_middleware.py`) and the
|
|
35
|
+
legacy `autoutils-log` / `django-autoutils` libraries — into one installable,
|
|
36
|
+
tested package with **zero required dependencies**. Django and `jdatetime`
|
|
37
|
+
are optional extras; the core works in any Python service.
|
|
38
|
+
|
|
39
|
+
## Install
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
uv add nilva-django-logger[django,jalali] # Django project
|
|
43
|
+
uv add nilva-django-logger # plain-Python service
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Or straight from GitLab while unpublished:
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
uv add "nilva-django-logger[django,jalali] @ git+ssh://git@gitlab.nilva.ai:2220/nilva/dev-assets/django/nilva-django-logger.git@develop"
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Quickstart (Django)
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
# settings.py
|
|
56
|
+
from nilva_django_logger import setup_logging
|
|
57
|
+
|
|
58
|
+
LOGGING_CONFIG = None # we configure logging ourselves
|
|
59
|
+
setup_logging(
|
|
60
|
+
app_name="myapp",
|
|
61
|
+
log_dir=BASE_DIR / "log", # /app/log in the container → tailed by Filebeat
|
|
62
|
+
log_name="myapp",
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
MIDDLEWARE = [
|
|
66
|
+
# ...
|
|
67
|
+
"django.contrib.auth.middleware.AuthenticationMiddleware",
|
|
68
|
+
"nilva_django_logger.middleware.RequestLoggerMiddleware",
|
|
69
|
+
# ...
|
|
70
|
+
]
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
# urls.py
|
|
75
|
+
from nilva_django_logger.health import health_urlpatterns
|
|
76
|
+
|
|
77
|
+
urlpatterns = [
|
|
78
|
+
# ...
|
|
79
|
+
*health_urlpatterns(), # /livez/ /readyz/ /startupz/
|
|
80
|
+
]
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
No `INSTALLED_APPS` entry is needed — the package has no models, templates or
|
|
84
|
+
signals.
|
|
85
|
+
|
|
86
|
+
What you get:
|
|
87
|
+
|
|
88
|
+
- **Console** — tidy coloured lines with Jalali timestamps (when `jdatetime`
|
|
89
|
+
is installed): `<dim ts> ● LEVEL <dim logger file:line> message`.
|
|
90
|
+
- **`log/myapp.log`** — rotating (10 MB × 10) house-format JSON, one complete
|
|
91
|
+
object per line, `ensure_ascii=False` so Persian stays readable:
|
|
92
|
+
|
|
93
|
+
```json
|
|
94
|
+
{"app_name": "myapp", "short_message": "...", "logger_data": {...}, "additional_data": {...}}
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
- **Request logging** — one structured line per request (method, path, status,
|
|
98
|
+
response time, user, client IP, view name, correlation id); on 4xx/5xx the
|
|
99
|
+
capped request/response bodies are attached for debugging. Health probes are
|
|
100
|
+
skipped entirely.
|
|
101
|
+
- **Secret masking everywhere** — `Bearer`/`Basic` credentials,
|
|
102
|
+
`password=...` / `"token": "..."` pairs and `<password>` XML tags are
|
|
103
|
+
redacted in messages *and* inside `extra={...}` payloads before any handler
|
|
104
|
+
writes them.
|
|
105
|
+
- **Quiet defaults** — `django.server` at WARNING (+ probe filter),
|
|
106
|
+
`PIL`/`urllib3`/`elasticsearch`/`kafka` tamed.
|
|
107
|
+
|
|
108
|
+
### Enriching request logs from views
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
from nilva_django_logger import set_log_data, set_log_message
|
|
112
|
+
|
|
113
|
+
def play(request):
|
|
114
|
+
...
|
|
115
|
+
set_log_message(request, "Play daily game") # replaces the generic summary line
|
|
116
|
+
set_log_data(request, score=score) # extra structured fields
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
### Dedicated flat request-log file (Fluentd/Grafana style)
|
|
120
|
+
|
|
121
|
+
The Bubble backend ships a flat JSON document per API call (`panel.log`) for
|
|
122
|
+
dashboards. Enable the same with:
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
setup_logging(..., request_file=True) # → log/myapp_request.log
|
|
126
|
+
setup_logging(..., request_file="panel.log") # custom name
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
### Kubernetes stdout mode
|
|
130
|
+
|
|
131
|
+
When the platform scrapes container stdout instead of files (as Bubble does),
|
|
132
|
+
ship JSON on the console and skip files entirely:
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
setup_logging(app_name="myapp", console_json=True)
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
## Settings
|
|
139
|
+
|
|
140
|
+
Everything can also be configured centrally via a `NILVA_LOGGER` dict in
|
|
141
|
+
Django settings (the `nilva-django-auth` pattern; `@override_settings`
|
|
142
|
+
reloads it in tests). Keyword arguments to `setup_logging()` /
|
|
143
|
+
`build_logging_config()` win over `NILVA_LOGGER`, which wins over defaults.
|
|
144
|
+
|
|
145
|
+
| Key | Default | Meaning |
|
|
146
|
+
| --- | --- | --- |
|
|
147
|
+
| `APP_NAME` | `None` | stamped on every JSON document |
|
|
148
|
+
| `HOST_NAME` | `None` | `"auto"` → `socket.gethostname()` |
|
|
149
|
+
| `LOG_DIR` | `None` | file logging directory (`None` = console only) |
|
|
150
|
+
| `LOG_NAME` | `"app"` | `<LOG_NAME>.log` file name |
|
|
151
|
+
| `MAX_BYTES` / `BACKUP_COUNT` | 10 MB / 10 | rotation (~110 MB ceiling per process) |
|
|
152
|
+
| `ROOT_LEVEL` / `CONSOLE_LEVEL` / `FILE_LEVEL` | DEBUG / INFO / INFO | levels |
|
|
153
|
+
| `REQUEST_LOGGER_NAME` | `"nilva_request"` | logger the middleware emits on (doesn't propagate to root) |
|
|
154
|
+
| `MAX_BODY_LOG` | 4096 | cap on logged request/response bodies |
|
|
155
|
+
| `LOG_BODIES_ON_ERROR` | `True` | attach bodies on 4xx/5xx |
|
|
156
|
+
| `BASE_URL_PREFIX` | `""` | strip before ADMIN/API/OTHER classification |
|
|
157
|
+
| `HEALTH_PROBE_PATHS` | `/livez/ /readyz/ /startupz/ /healthz` | skipped by request logging + `HealthProbeFilter` |
|
|
158
|
+
| `MASK` | `"*****"` | replacement text |
|
|
159
|
+
| `SENSITIVE_KEYS` | built-in set | extra-payload key names masked wholesale |
|
|
160
|
+
| `EXTRA_MASK_PATTERNS` | `()` | additional regexes (group 1 = kept prefix) |
|
|
161
|
+
| `MASK_EXTRAS` | `True` | also mask inside `extra={...}` values |
|
|
162
|
+
| `JALALI_CONSOLE` | `True` | Jalali console timestamps when `jdatetime` present |
|
|
163
|
+
|
|
164
|
+
**Config-time vs. runtime.** `NILVA_LOGGER` values are read live and honored
|
|
165
|
+
once Django settings are configured. If you call `setup_logging()` at the
|
|
166
|
+
bottom of `settings.py` (the usual place), Django is still mid-configuration at
|
|
167
|
+
that moment, so handler/formatter options baked in right then — `APP_NAME`,
|
|
168
|
+
`LOG_DIR`, levels, `MASK` — should be passed as `setup_logging()` **keyword
|
|
169
|
+
arguments**. Middleware and request-time options (`HEALTH_PROBE_PATHS`,
|
|
170
|
+
`BASE_URL_PREFIX`, `LOG_BODIES_ON_ERROR`, `MAX_BODY_LOG`) are read per request
|
|
171
|
+
and honor `NILVA_LOGGER` directly.
|
|
172
|
+
|
|
173
|
+
## Masking coverage
|
|
174
|
+
|
|
175
|
+
`MaskSensitiveDataFilter` redacts secrets in the message text (`Bearer`/`Basic`
|
|
176
|
+
credentials, `key=value` pairs and quoted `key: value` pairs in either quote
|
|
177
|
+
style — including mixed quoting like `"password": 'x'` — and `<key>` XML tags;
|
|
178
|
+
quoted values containing spaces are masked in full) **and** recursively
|
|
179
|
+
inside `extra={...}` payloads (values under a sensitive key name are replaced
|
|
180
|
+
wholesale; deeper-than-8 or cyclic structures fail closed to the mask). Because
|
|
181
|
+
regexing serialized text is best-effort, **prefer structured `extra={...}`** for
|
|
182
|
+
anything sensitive — that path masks by key and never depends on quoting.
|
|
183
|
+
|
|
184
|
+
## Plain-Python (no Django)
|
|
185
|
+
|
|
186
|
+
The formatters, filters and config builder are stdlib-only:
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
import logging
|
|
190
|
+
from nilva_django_logger import setup_logging
|
|
191
|
+
|
|
192
|
+
setup_logging(app_name="worker", log_dir="/app/log", log_name="worker")
|
|
193
|
+
logging.getLogger("worker").info("processed", extra={"job_id": 42})
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
Or hand-wire single pieces into an existing config:
|
|
197
|
+
|
|
198
|
+
```python
|
|
199
|
+
from nilva_django_logger import JsonFormatter, MaskSensitiveDataFilter
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
## Startup banner
|
|
203
|
+
|
|
204
|
+
```python
|
|
205
|
+
# gunicorn.conf.py
|
|
206
|
+
def on_starting(server):
|
|
207
|
+
from nilva_django_logger import print_banner
|
|
208
|
+
print_banner(MYAPP_FIGLET, subtitle=f"gunicorn · {bind} · {workers} workers")
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
Generate the art once with `figlet -f "ANSI Shadow" MYAPP`.
|
|
212
|
+
|
|
213
|
+
## Deployment conventions (Filebeat flow)
|
|
214
|
+
|
|
215
|
+
- In the container the app lives at `/app`, so logs land in **`/app/log/*.log`**
|
|
216
|
+
where the Filebeat sidecar tails them. Pre-create the directory in the
|
|
217
|
+
Dockerfile **owned by the non-root runtime user**; `setup_logging` also
|
|
218
|
+
creates it on the fly for local runs and falls back to console-only if the
|
|
219
|
+
directory is unwritable (an unwritable log dir must never take the app down).
|
|
220
|
+
- Add `log/` to `.gitignore` **and** `.dockerignore`.
|
|
221
|
+
- Wire the probes into your Deployment (`livez` never touches the DB — a DB
|
|
222
|
+
outage must not restart pods; `readyz`/`startupz` check the DB).
|
|
223
|
+
|
|
224
|
+
## Design notes
|
|
225
|
+
|
|
226
|
+
- The house JSON shape (`short_message` / `logger_data` / `additional_data`)
|
|
227
|
+
comes from `autoutils-log`'s `BaseLogHandler`; `app_name`/`host_name` from
|
|
228
|
+
its handler kwargs and Bubble; the `exception` key, `ensure_ascii=False`
|
|
229
|
+
and the writable-dir fallback from Shahab.
|
|
230
|
+
- Filebeat/ECS fields (`log`, `docker`, `host`, `container`, `ecs`, `agent`,
|
|
231
|
+
`stream`, `input`) are never re-emitted as `additional_data` (a Bubble
|
|
232
|
+
lesson — re-ingested lines double-enrich).
|
|
233
|
+
- `autoutils-log`'s `LogstashHandler`/`KafkaHandler` (blocking network sends
|
|
234
|
+
per record) were deliberately **not** ported: the house flow is rotating
|
|
235
|
+
files + Filebeat. `ColorfulStreamHandler`'s `termcolor`/`pytz`/
|
|
236
|
+
`persiantools` dependencies are gone — ANSI codes and `jdatetime` (optional)
|
|
237
|
+
suffice.
|
|
238
|
+
|
|
239
|
+
## Development
|
|
240
|
+
|
|
241
|
+
```bash
|
|
242
|
+
uv sync --group dev
|
|
243
|
+
./run_tests.sh # pytest
|
|
244
|
+
./run_tests.sh -c # with coverage
|
|
245
|
+
uv run ruff check .
|
|
246
|
+
```
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
# nilva-django-logger
|
|
2
|
+
|
|
3
|
+
The Nilva house logging toolkit — structured one-JSON-object-per-line logs
|
|
4
|
+
(shipped by Filebeat), a colourful self-contained console formatter, secret
|
|
5
|
+
masking, request/response logging middleware, Kubernetes health probes and
|
|
6
|
+
probe-noise filtering.
|
|
7
|
+
|
|
8
|
+
It consolidates the logging stack that grew across Nilva backends —
|
|
9
|
+
`shahab-backend` (`shahab/log_formatter.py`, `settings/logging.py`,
|
|
10
|
+
`settings/log_filter.py`, `health.py`, `banner.py`), `mymci-bubble-backend`
|
|
11
|
+
(`core/log_formatter.py`, `core/middlewares/logger_middleware.py`) and the
|
|
12
|
+
legacy `autoutils-log` / `django-autoutils` libraries — into one installable,
|
|
13
|
+
tested package with **zero required dependencies**. Django and `jdatetime`
|
|
14
|
+
are optional extras; the core works in any Python service.
|
|
15
|
+
|
|
16
|
+
## Install
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
uv add nilva-django-logger[django,jalali] # Django project
|
|
20
|
+
uv add nilva-django-logger # plain-Python service
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Or straight from GitLab while unpublished:
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
uv add "nilva-django-logger[django,jalali] @ git+ssh://git@gitlab.nilva.ai:2220/nilva/dev-assets/django/nilva-django-logger.git@develop"
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
## Quickstart (Django)
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
# settings.py
|
|
33
|
+
from nilva_django_logger import setup_logging
|
|
34
|
+
|
|
35
|
+
LOGGING_CONFIG = None # we configure logging ourselves
|
|
36
|
+
setup_logging(
|
|
37
|
+
app_name="myapp",
|
|
38
|
+
log_dir=BASE_DIR / "log", # /app/log in the container → tailed by Filebeat
|
|
39
|
+
log_name="myapp",
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
MIDDLEWARE = [
|
|
43
|
+
# ...
|
|
44
|
+
"django.contrib.auth.middleware.AuthenticationMiddleware",
|
|
45
|
+
"nilva_django_logger.middleware.RequestLoggerMiddleware",
|
|
46
|
+
# ...
|
|
47
|
+
]
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
# urls.py
|
|
52
|
+
from nilva_django_logger.health import health_urlpatterns
|
|
53
|
+
|
|
54
|
+
urlpatterns = [
|
|
55
|
+
# ...
|
|
56
|
+
*health_urlpatterns(), # /livez/ /readyz/ /startupz/
|
|
57
|
+
]
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
No `INSTALLED_APPS` entry is needed — the package has no models, templates or
|
|
61
|
+
signals.
|
|
62
|
+
|
|
63
|
+
What you get:
|
|
64
|
+
|
|
65
|
+
- **Console** — tidy coloured lines with Jalali timestamps (when `jdatetime`
|
|
66
|
+
is installed): `<dim ts> ● LEVEL <dim logger file:line> message`.
|
|
67
|
+
- **`log/myapp.log`** — rotating (10 MB × 10) house-format JSON, one complete
|
|
68
|
+
object per line, `ensure_ascii=False` so Persian stays readable:
|
|
69
|
+
|
|
70
|
+
```json
|
|
71
|
+
{"app_name": "myapp", "short_message": "...", "logger_data": {...}, "additional_data": {...}}
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
- **Request logging** — one structured line per request (method, path, status,
|
|
75
|
+
response time, user, client IP, view name, correlation id); on 4xx/5xx the
|
|
76
|
+
capped request/response bodies are attached for debugging. Health probes are
|
|
77
|
+
skipped entirely.
|
|
78
|
+
- **Secret masking everywhere** — `Bearer`/`Basic` credentials,
|
|
79
|
+
`password=...` / `"token": "..."` pairs and `<password>` XML tags are
|
|
80
|
+
redacted in messages *and* inside `extra={...}` payloads before any handler
|
|
81
|
+
writes them.
|
|
82
|
+
- **Quiet defaults** — `django.server` at WARNING (+ probe filter),
|
|
83
|
+
`PIL`/`urllib3`/`elasticsearch`/`kafka` tamed.
|
|
84
|
+
|
|
85
|
+
### Enriching request logs from views
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
from nilva_django_logger import set_log_data, set_log_message
|
|
89
|
+
|
|
90
|
+
def play(request):
|
|
91
|
+
...
|
|
92
|
+
set_log_message(request, "Play daily game") # replaces the generic summary line
|
|
93
|
+
set_log_data(request, score=score) # extra structured fields
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
### Dedicated flat request-log file (Fluentd/Grafana style)
|
|
97
|
+
|
|
98
|
+
The Bubble backend ships a flat JSON document per API call (`panel.log`) for
|
|
99
|
+
dashboards. Enable the same with:
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
setup_logging(..., request_file=True) # → log/myapp_request.log
|
|
103
|
+
setup_logging(..., request_file="panel.log") # custom name
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
### Kubernetes stdout mode
|
|
107
|
+
|
|
108
|
+
When the platform scrapes container stdout instead of files (as Bubble does),
|
|
109
|
+
ship JSON on the console and skip files entirely:
|
|
110
|
+
|
|
111
|
+
```python
|
|
112
|
+
setup_logging(app_name="myapp", console_json=True)
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
## Settings
|
|
116
|
+
|
|
117
|
+
Everything can also be configured centrally via a `NILVA_LOGGER` dict in
|
|
118
|
+
Django settings (the `nilva-django-auth` pattern; `@override_settings`
|
|
119
|
+
reloads it in tests). Keyword arguments to `setup_logging()` /
|
|
120
|
+
`build_logging_config()` win over `NILVA_LOGGER`, which wins over defaults.
|
|
121
|
+
|
|
122
|
+
| Key | Default | Meaning |
|
|
123
|
+
| --- | --- | --- |
|
|
124
|
+
| `APP_NAME` | `None` | stamped on every JSON document |
|
|
125
|
+
| `HOST_NAME` | `None` | `"auto"` → `socket.gethostname()` |
|
|
126
|
+
| `LOG_DIR` | `None` | file logging directory (`None` = console only) |
|
|
127
|
+
| `LOG_NAME` | `"app"` | `<LOG_NAME>.log` file name |
|
|
128
|
+
| `MAX_BYTES` / `BACKUP_COUNT` | 10 MB / 10 | rotation (~110 MB ceiling per process) |
|
|
129
|
+
| `ROOT_LEVEL` / `CONSOLE_LEVEL` / `FILE_LEVEL` | DEBUG / INFO / INFO | levels |
|
|
130
|
+
| `REQUEST_LOGGER_NAME` | `"nilva_request"` | logger the middleware emits on (doesn't propagate to root) |
|
|
131
|
+
| `MAX_BODY_LOG` | 4096 | cap on logged request/response bodies |
|
|
132
|
+
| `LOG_BODIES_ON_ERROR` | `True` | attach bodies on 4xx/5xx |
|
|
133
|
+
| `BASE_URL_PREFIX` | `""` | strip before ADMIN/API/OTHER classification |
|
|
134
|
+
| `HEALTH_PROBE_PATHS` | `/livez/ /readyz/ /startupz/ /healthz` | skipped by request logging + `HealthProbeFilter` |
|
|
135
|
+
| `MASK` | `"*****"` | replacement text |
|
|
136
|
+
| `SENSITIVE_KEYS` | built-in set | extra-payload key names masked wholesale |
|
|
137
|
+
| `EXTRA_MASK_PATTERNS` | `()` | additional regexes (group 1 = kept prefix) |
|
|
138
|
+
| `MASK_EXTRAS` | `True` | also mask inside `extra={...}` values |
|
|
139
|
+
| `JALALI_CONSOLE` | `True` | Jalali console timestamps when `jdatetime` present |
|
|
140
|
+
|
|
141
|
+
**Config-time vs. runtime.** `NILVA_LOGGER` values are read live and honored
|
|
142
|
+
once Django settings are configured. If you call `setup_logging()` at the
|
|
143
|
+
bottom of `settings.py` (the usual place), Django is still mid-configuration at
|
|
144
|
+
that moment, so handler/formatter options baked in right then — `APP_NAME`,
|
|
145
|
+
`LOG_DIR`, levels, `MASK` — should be passed as `setup_logging()` **keyword
|
|
146
|
+
arguments**. Middleware and request-time options (`HEALTH_PROBE_PATHS`,
|
|
147
|
+
`BASE_URL_PREFIX`, `LOG_BODIES_ON_ERROR`, `MAX_BODY_LOG`) are read per request
|
|
148
|
+
and honor `NILVA_LOGGER` directly.
|
|
149
|
+
|
|
150
|
+
## Masking coverage
|
|
151
|
+
|
|
152
|
+
`MaskSensitiveDataFilter` redacts secrets in the message text (`Bearer`/`Basic`
|
|
153
|
+
credentials, `key=value` pairs and quoted `key: value` pairs in either quote
|
|
154
|
+
style — including mixed quoting like `"password": 'x'` — and `<key>` XML tags;
|
|
155
|
+
quoted values containing spaces are masked in full) **and** recursively
|
|
156
|
+
inside `extra={...}` payloads (values under a sensitive key name are replaced
|
|
157
|
+
wholesale; deeper-than-8 or cyclic structures fail closed to the mask). Because
|
|
158
|
+
regexing serialized text is best-effort, **prefer structured `extra={...}`** for
|
|
159
|
+
anything sensitive — that path masks by key and never depends on quoting.
|
|
160
|
+
|
|
161
|
+
## Plain-Python (no Django)
|
|
162
|
+
|
|
163
|
+
The formatters, filters and config builder are stdlib-only:
|
|
164
|
+
|
|
165
|
+
```python
|
|
166
|
+
import logging
|
|
167
|
+
from nilva_django_logger import setup_logging
|
|
168
|
+
|
|
169
|
+
setup_logging(app_name="worker", log_dir="/app/log", log_name="worker")
|
|
170
|
+
logging.getLogger("worker").info("processed", extra={"job_id": 42})
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
Or hand-wire single pieces into an existing config:
|
|
174
|
+
|
|
175
|
+
```python
|
|
176
|
+
from nilva_django_logger import JsonFormatter, MaskSensitiveDataFilter
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
## Startup banner
|
|
180
|
+
|
|
181
|
+
```python
|
|
182
|
+
# gunicorn.conf.py
|
|
183
|
+
def on_starting(server):
|
|
184
|
+
from nilva_django_logger import print_banner
|
|
185
|
+
print_banner(MYAPP_FIGLET, subtitle=f"gunicorn · {bind} · {workers} workers")
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
Generate the art once with `figlet -f "ANSI Shadow" MYAPP`.
|
|
189
|
+
|
|
190
|
+
## Deployment conventions (Filebeat flow)
|
|
191
|
+
|
|
192
|
+
- In the container the app lives at `/app`, so logs land in **`/app/log/*.log`**
|
|
193
|
+
where the Filebeat sidecar tails them. Pre-create the directory in the
|
|
194
|
+
Dockerfile **owned by the non-root runtime user**; `setup_logging` also
|
|
195
|
+
creates it on the fly for local runs and falls back to console-only if the
|
|
196
|
+
directory is unwritable (an unwritable log dir must never take the app down).
|
|
197
|
+
- Add `log/` to `.gitignore` **and** `.dockerignore`.
|
|
198
|
+
- Wire the probes into your Deployment (`livez` never touches the DB — a DB
|
|
199
|
+
outage must not restart pods; `readyz`/`startupz` check the DB).
|
|
200
|
+
|
|
201
|
+
## Design notes
|
|
202
|
+
|
|
203
|
+
- The house JSON shape (`short_message` / `logger_data` / `additional_data`)
|
|
204
|
+
comes from `autoutils-log`'s `BaseLogHandler`; `app_name`/`host_name` from
|
|
205
|
+
its handler kwargs and Bubble; the `exception` key, `ensure_ascii=False`
|
|
206
|
+
and the writable-dir fallback from Shahab.
|
|
207
|
+
- Filebeat/ECS fields (`log`, `docker`, `host`, `container`, `ecs`, `agent`,
|
|
208
|
+
`stream`, `input`) are never re-emitted as `additional_data` (a Bubble
|
|
209
|
+
lesson — re-ingested lines double-enrich).
|
|
210
|
+
- `autoutils-log`'s `LogstashHandler`/`KafkaHandler` (blocking network sends
|
|
211
|
+
per record) were deliberately **not** ported: the house flow is rotating
|
|
212
|
+
files + Filebeat. `ColorfulStreamHandler`'s `termcolor`/`pytz`/
|
|
213
|
+
`persiantools` dependencies are gone — ANSI codes and `jdatetime` (optional)
|
|
214
|
+
suffice.
|
|
215
|
+
|
|
216
|
+
## Development
|
|
217
|
+
|
|
218
|
+
```bash
|
|
219
|
+
uv sync --group dev
|
|
220
|
+
./run_tests.sh # pytest
|
|
221
|
+
./run_tests.sh -c # with coverage
|
|
222
|
+
uv run ruff check .
|
|
223
|
+
```
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Nilva Django Logger — the Nilva house logging toolkit.
|
|
2
|
+
|
|
3
|
+
Structured one-JSON-object-per-line logs (shipped by Filebeat), a colourful
|
|
4
|
+
self-contained console formatter, secret masking, request/response logging
|
|
5
|
+
middleware, Kubernetes health probes and probe-noise filtering.
|
|
6
|
+
|
|
7
|
+
The core (formatters, filters, config builder, banner) is pure standard
|
|
8
|
+
library and works in any Python service; the Django pieces (middleware,
|
|
9
|
+
health views, ``NILVA_LOGGER`` settings) load lazily so importing this
|
|
10
|
+
package never requires Django.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
__version__ = "0.1.4"
|
|
14
|
+
|
|
15
|
+
from .banner import print_banner, render_banner
|
|
16
|
+
from .config import build_logging_config, setup_logging
|
|
17
|
+
from .filters import HealthProbeFilter, MaskSensitiveDataFilter
|
|
18
|
+
from .formatters import ColorfulFormatter, JsonFormatter, RequestJsonFormatter
|
|
19
|
+
from .settings import logger_settings
|
|
20
|
+
|
|
21
|
+
#: Django-only exports, resolved lazily so ``import nilva_django_logger``
|
|
22
|
+
#: works in plain-Python services without Django installed.
|
|
23
|
+
_DJANGO_EXPORTS = {
|
|
24
|
+
"RequestLoggerMiddleware": "middleware",
|
|
25
|
+
"set_log_message": "middleware",
|
|
26
|
+
"set_log_data": "middleware",
|
|
27
|
+
"get_client_ip": "middleware",
|
|
28
|
+
"livez": "health",
|
|
29
|
+
"readyz": "health",
|
|
30
|
+
"startupz": "health",
|
|
31
|
+
"health_urlpatterns": "health",
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
"__version__",
|
|
36
|
+
"JsonFormatter",
|
|
37
|
+
"ColorfulFormatter",
|
|
38
|
+
"RequestJsonFormatter",
|
|
39
|
+
"MaskSensitiveDataFilter",
|
|
40
|
+
"HealthProbeFilter",
|
|
41
|
+
"build_logging_config",
|
|
42
|
+
"setup_logging",
|
|
43
|
+
"render_banner",
|
|
44
|
+
"print_banner",
|
|
45
|
+
"logger_settings",
|
|
46
|
+
*_DJANGO_EXPORTS,
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def __getattr__(name):
|
|
51
|
+
module_name = _DJANGO_EXPORTS.get(name)
|
|
52
|
+
if module_name is None:
|
|
53
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
54
|
+
from importlib import import_module
|
|
55
|
+
|
|
56
|
+
module = import_module(f".{module_name}", __name__)
|
|
57
|
+
value = getattr(module, name)
|
|
58
|
+
globals()[name] = value
|
|
59
|
+
return value
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""Startup banner — print big ASCII art to the terminal when the server boots.
|
|
2
|
+
|
|
3
|
+
Give it your project's figlet (``figlet -f "ANSI Shadow" MYAPP``) and wire it
|
|
4
|
+
into gunicorn's ``on_starting`` hook (``gunicorn.conf.py``) so it shows once,
|
|
5
|
+
in the terminal / container logs, right as the project comes up::
|
|
6
|
+
|
|
7
|
+
# gunicorn.conf.py
|
|
8
|
+
def on_starting(server):
|
|
9
|
+
from myapp.art import ART # the figlet string
|
|
10
|
+
from nilva_django_logger import print_banner
|
|
11
|
+
|
|
12
|
+
print_banner(ART, subtitle=f"gunicorn · {bind} · {workers} workers")
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import sys
|
|
18
|
+
|
|
19
|
+
_RESET = "\033[0m"
|
|
20
|
+
_BOLD = "\033[1m"
|
|
21
|
+
_DIM = "\033[2m"
|
|
22
|
+
|
|
23
|
+
_COLORS = {
|
|
24
|
+
"cyan": "\033[36m",
|
|
25
|
+
"magenta": "\033[35m",
|
|
26
|
+
"yellow": "\033[33m",
|
|
27
|
+
"green": "\033[32m",
|
|
28
|
+
"blue": "\033[34m",
|
|
29
|
+
"red": "\033[31m",
|
|
30
|
+
"white": "\033[37m",
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def render_banner(
|
|
35
|
+
art: str,
|
|
36
|
+
subtitle: str | None = None,
|
|
37
|
+
*,
|
|
38
|
+
color: str = "cyan",
|
|
39
|
+
use_colors: bool = True,
|
|
40
|
+
) -> str:
|
|
41
|
+
"""Return the coloured banner (+ optional dimmed subtitle line)."""
|
|
42
|
+
art = art.strip("\n")
|
|
43
|
+
lines = []
|
|
44
|
+
if use_colors:
|
|
45
|
+
lines.append(f"{_BOLD}{_COLORS.get(color, _COLORS['cyan'])}{art}{_RESET}")
|
|
46
|
+
if subtitle:
|
|
47
|
+
lines.append(f"{_DIM} {subtitle}{_RESET}")
|
|
48
|
+
else:
|
|
49
|
+
lines.append(art)
|
|
50
|
+
if subtitle:
|
|
51
|
+
lines.append(f" {subtitle}")
|
|
52
|
+
return "\n".join(lines)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def print_banner(art: str, subtitle: str | None = None, **kwargs) -> None:
|
|
56
|
+
"""Print the banner to stdout (the terminal / container log)."""
|
|
57
|
+
sys.stdout.write("\n" + render_banner(art, subtitle, **kwargs) + "\n\n")
|
|
58
|
+
sys.stdout.flush()
|