autoshiploads-logging 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- autoshiploads_logging-1.0.0/PKG-INFO +179 -0
- autoshiploads_logging-1.0.0/README.md +165 -0
- autoshiploads_logging-1.0.0/pyproject.toml +27 -0
- autoshiploads_logging-1.0.0/src/autoshiploads_logging/__init__.py +49 -0
- autoshiploads_logging-1.0.0/src/autoshiploads_logging/_events.py +52 -0
- autoshiploads_logging-1.0.0/src/autoshiploads_logging/_format.py +194 -0
- autoshiploads_logging-1.0.0/src/autoshiploads_logging/_handlers.py +73 -0
- autoshiploads_logging-1.0.0/src/autoshiploads_logging/_job.py +61 -0
- autoshiploads_logging-1.0.0/src/autoshiploads_logging/_logger.py +113 -0
- autoshiploads_logging-1.0.0/src/autoshiploads_logging/_retention.py +80 -0
- autoshiploads_logging-1.0.0/src/autoshiploads_logging/_sanitize.py +101 -0
- autoshiploads_logging-1.0.0/src/autoshiploads_logging/_setup.py +282 -0
- autoshiploads_logging-1.0.0/src/autoshiploads_logging/_time.py +42 -0
- autoshiploads_logging-1.0.0/src/autoshiploads_logging/_trace.py +46 -0
- autoshiploads_logging-1.0.0/src/autoshiploads_logging/_version.py +11 -0
- autoshiploads_logging-1.0.0/src/autoshiploads_logging/_writer.py +84 -0
- autoshiploads_logging-1.0.0/src/autoshiploads_logging/django.py +70 -0
- autoshiploads_logging-1.0.0/src/autoshiploads_logging/py.typed +0 -0
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: autoshiploads-logging
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: JSONL logging for Shiploads Python apps: the logging spec's line (ECS names), weekly files named by the Monday in Hobart, retention, event catalogue, trace ids, Django request logging.
|
|
5
|
+
Keywords: logging,jsonl,ecs,django
|
|
6
|
+
License-Expression: LicenseRef-Proprietary
|
|
7
|
+
Classifier: Programming Language :: Python :: 3
|
|
8
|
+
Classifier: Framework :: Django
|
|
9
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
10
|
+
Classifier: Operating System :: POSIX
|
|
11
|
+
Requires-Dist: tzdata ; sys_platform == 'win32'
|
|
12
|
+
Requires-Python: >=3.11
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
|
|
15
|
+
# autoshiploads-logging
|
|
16
|
+
|
|
17
|
+
The Shiploads logging spec for Python. Every app writes the same JSON line to the same weekly
|
|
18
|
+
files as the Node apps (`@autoshiploads/logging`), so one log viewer can read them all. It
|
|
19
|
+
plugs into Python's built-in `logging`, so Django, waitress and APScheduler lines land in the
|
|
20
|
+
same file, tagged by where they came from.
|
|
21
|
+
|
|
22
|
+
```
|
|
23
|
+
app code, Django, waitress, APScheduler
|
|
24
|
+
|
|
|
25
|
+
v
|
|
26
|
+
stdlib logging (root logger)
|
|
27
|
+
|
|
|
28
|
+
v
|
|
29
|
+
one JSON line (ECS field names)
|
|
30
|
+
|
|
|
31
|
+
+--> logs/<service>_<Monday>.jsonl
|
|
32
|
+
| weekly, Hobart; 90 days kept
|
|
33
|
+
|
|
|
34
|
+
+--> logs/<service>.<stream>_<Monday>.jsonl
|
|
35
|
+
| optional longer-kept stream
|
|
36
|
+
|
|
|
37
|
+
+--> readable line on stderr
|
|
38
|
+
(only in a terminal)
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Install
|
|
42
|
+
|
|
43
|
+
```
|
|
44
|
+
uv add autoshiploads-logging==1.0.0
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Python 3.11 or later. No dependencies except `tzdata` on Windows (time zone data, which Django
|
|
48
|
+
already brings).
|
|
49
|
+
|
|
50
|
+
## Django
|
|
51
|
+
|
|
52
|
+
`settings.py`: hand logging to the package, and put the middleware first.
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
LOGGING_CONFIG = None # Django doesn't configure logging; the package does
|
|
56
|
+
|
|
57
|
+
MIDDLEWARE = [
|
|
58
|
+
"autoshiploads_logging.django.RequestLoggingMiddleware",
|
|
59
|
+
# ... the rest
|
|
60
|
+
]
|
|
61
|
+
|
|
62
|
+
AUTOSHIPLOADS_LOGGING = {
|
|
63
|
+
"ignore_paths": ["/health/"],
|
|
64
|
+
"trust_x_forwarded_for": True, # behind IIS, so client.ip is the caller
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
from autoshiploads_logging import setup_logging # at the end of settings.py
|
|
68
|
+
|
|
69
|
+
setup_logging("trackpallet", "3.1.0")
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Each request gets one `http_request` line (path only, never the query string) and a trace id
|
|
73
|
+
that every line logged while handling it carries, Django's own error lines included. The id
|
|
74
|
+
comes from the caller's `X-Request-Id` when it's safe, and is sent back in that header.
|
|
75
|
+
|
|
76
|
+
## The catalogue and logging
|
|
77
|
+
|
|
78
|
+
One module in the app names every business event:
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
# trackpallet/logging_events.py
|
|
82
|
+
from autoshiploads_logging import define_events
|
|
83
|
+
|
|
84
|
+
events = define_events(
|
|
85
|
+
pallet_recorded="business",
|
|
86
|
+
weekly_report_sent="business",
|
|
87
|
+
)
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from autoshiploads_logging import get_logger
|
|
92
|
+
from .logging_events import events
|
|
93
|
+
|
|
94
|
+
log = get_logger(__name__)
|
|
95
|
+
|
|
96
|
+
log.event(events.pallet_recorded, f"Pallet recorded: {store}", {"store": store, "pallets": n})
|
|
97
|
+
log.info("Report email queued") # category system
|
|
98
|
+
log.error("CI query failed", error=exc, data={"store": "CAM"})
|
|
99
|
+
log.event(events.weekly_report_sent, "...", data, level="error", duration_ms=ms)
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Plain `logging.getLogger(...)` calls keep working and produce valid lines; any `extra=` values
|
|
103
|
+
become `data`. Names are snake_case; `run_started`, `run_finished`, `http_request` and
|
|
104
|
+
`log_files_deleted` are written by the package. Categories: `request`, `business`, `job`,
|
|
105
|
+
`system`.
|
|
106
|
+
|
|
107
|
+
## Jobs, commands and scripts
|
|
108
|
+
|
|
109
|
+
A fresh trace id per run, and `run_started` / `run_finished` with the result and duration. A
|
|
110
|
+
failure is logged at `error` and re-raised; `SystemExit(0)` counts as success.
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
from autoshiploads_logging import run_job
|
|
114
|
+
|
|
115
|
+
with run_job(log, "weekly_report"):
|
|
116
|
+
...
|
|
117
|
+
|
|
118
|
+
@run_job(log, "nightly_import") # APScheduler job, management command
|
|
119
|
+
def nightly_import(): ...
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
A script outside an app calls `setup_logging("<script>", "<version>",
|
|
123
|
+
log_dir=r"C:\ProgramData\Shiploads\Logs\<script>")` first (spec section 5).
|
|
124
|
+
|
|
125
|
+
## A longer-kept stream
|
|
126
|
+
|
|
127
|
+
For events people ask about long after (COB's picks, kept a year): those loggers write to their
|
|
128
|
+
own file with their own retention, and not to the main file.
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from autoshiploads_logging import StreamConfig, setup_logging
|
|
132
|
+
|
|
133
|
+
setup_logging(
|
|
134
|
+
"cob",
|
|
135
|
+
"2.4.0",
|
|
136
|
+
streams={"picks": StreamConfig(loggers=("stock_movement",), retention_days=364)},
|
|
137
|
+
)
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
## Options
|
|
141
|
+
|
|
142
|
+
| Option | Default |
|
|
143
|
+
|---|---|
|
|
144
|
+
| `service`, `version` | required |
|
|
145
|
+
| `level` | `LOG_LEVEL` env, else `info` |
|
|
146
|
+
| `log_dir` | `LOG_DIR` env, else `<cwd>/logs` |
|
|
147
|
+
| `retention_days` | `LOG_RETENTION_DAYS`, else 90 |
|
|
148
|
+
| `time_zone` | `LOG_TIME_ZONE`, else Hobart |
|
|
149
|
+
| `output` | `file`; `stdout` for containers |
|
|
150
|
+
| `console` | `LOG_CONSOLE` 1/0, else terminal only |
|
|
151
|
+
| `redact_keys` | none (adds to the spec's list) |
|
|
152
|
+
| `streams` | none |
|
|
153
|
+
| `library_categories` | adds to the built-in map |
|
|
154
|
+
|
|
155
|
+
`LOG_LEVEL` wins over `level`, so a host can turn on `debug` without a code change. A bad value
|
|
156
|
+
from the environment is reported once as a `warn` line at startup and the default used.
|
|
157
|
+
|
|
158
|
+
## Files, retention, safety
|
|
159
|
+
|
|
160
|
+
- `logs/<service>_<YYYY-MM-DD>.jsonl`, dated by the Monday the week starts in Hobart; each line
|
|
161
|
+
picks its file from its own time, so rotation needs no timer.
|
|
162
|
+
- One write per line on an append-only handle, so several processes (waitress, a management
|
|
163
|
+
command, the dev reloader) can share a file without mixing lines. On Windows this opens the
|
|
164
|
+
file with `FILE_APPEND_DATA` directly: Python's own append mode goes through the C runtime,
|
|
165
|
+
which seeks and then writes in two steps, and in testing three processes writing one file
|
|
166
|
+
that way produced broken lines.
|
|
167
|
+
- Retention runs at setup and every 24 hours, by the date in the file name only: a weekly file
|
|
168
|
+
goes once its Sunday is past retention; WinSW's daily `*.yyyyMMdd.out.log` / `.err.log` once
|
|
169
|
+
their day is; a stream's files by the stream's own retention. Nothing else is touched.
|
|
170
|
+
- A failed write is never raised into the app; the first one prints a line to stderr.
|
|
171
|
+
- Keys in the spec's never-log list (`password`, `token`, `authorization`, `cookie`, ...) are
|
|
172
|
+
dropped from `data` at any depth. Objects JSON can't hold are written as their type name
|
|
173
|
+
only, never their repr (Django's request repr includes the query string).
|
|
174
|
+
|
|
175
|
+
## Changing this package
|
|
176
|
+
|
|
177
|
+
The spec (`~/.claude/logging-spec.md`) changes first, then this package (spec section 9).
|
|
178
|
+
Adding a field or option: minor version. Renaming, removing or retyping a field: major version.
|
|
179
|
+
See the repo README for releasing.
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# autoshiploads-logging
|
|
2
|
+
|
|
3
|
+
The Shiploads logging spec for Python. Every app writes the same JSON line to the same weekly
|
|
4
|
+
files as the Node apps (`@autoshiploads/logging`), so one log viewer can read them all. It
|
|
5
|
+
plugs into Python's built-in `logging`, so Django, waitress and APScheduler lines land in the
|
|
6
|
+
same file, tagged by where they came from.
|
|
7
|
+
|
|
8
|
+
```
|
|
9
|
+
app code, Django, waitress, APScheduler
|
|
10
|
+
|
|
|
11
|
+
v
|
|
12
|
+
stdlib logging (root logger)
|
|
13
|
+
|
|
|
14
|
+
v
|
|
15
|
+
one JSON line (ECS field names)
|
|
16
|
+
|
|
|
17
|
+
+--> logs/<service>_<Monday>.jsonl
|
|
18
|
+
| weekly, Hobart; 90 days kept
|
|
19
|
+
|
|
|
20
|
+
+--> logs/<service>.<stream>_<Monday>.jsonl
|
|
21
|
+
| optional longer-kept stream
|
|
22
|
+
|
|
|
23
|
+
+--> readable line on stderr
|
|
24
|
+
(only in a terminal)
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## Install
|
|
28
|
+
|
|
29
|
+
```
|
|
30
|
+
uv add autoshiploads-logging==1.0.0
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Python 3.11 or later. No dependencies except `tzdata` on Windows (time zone data, which Django
|
|
34
|
+
already brings).
|
|
35
|
+
|
|
36
|
+
## Django
|
|
37
|
+
|
|
38
|
+
`settings.py`: hand logging to the package, and put the middleware first.
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
LOGGING_CONFIG = None # Django doesn't configure logging; the package does
|
|
42
|
+
|
|
43
|
+
MIDDLEWARE = [
|
|
44
|
+
"autoshiploads_logging.django.RequestLoggingMiddleware",
|
|
45
|
+
# ... the rest
|
|
46
|
+
]
|
|
47
|
+
|
|
48
|
+
AUTOSHIPLOADS_LOGGING = {
|
|
49
|
+
"ignore_paths": ["/health/"],
|
|
50
|
+
"trust_x_forwarded_for": True, # behind IIS, so client.ip is the caller
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
from autoshiploads_logging import setup_logging # at the end of settings.py
|
|
54
|
+
|
|
55
|
+
setup_logging("trackpallet", "3.1.0")
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Each request gets one `http_request` line (path only, never the query string) and a trace id
|
|
59
|
+
that every line logged while handling it carries, Django's own error lines included. The id
|
|
60
|
+
comes from the caller's `X-Request-Id` when it's safe, and is sent back in that header.
|
|
61
|
+
|
|
62
|
+
## The catalogue and logging
|
|
63
|
+
|
|
64
|
+
One module in the app names every business event:
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
# trackpallet/logging_events.py
|
|
68
|
+
from autoshiploads_logging import define_events
|
|
69
|
+
|
|
70
|
+
events = define_events(
|
|
71
|
+
pallet_recorded="business",
|
|
72
|
+
weekly_report_sent="business",
|
|
73
|
+
)
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
from autoshiploads_logging import get_logger
|
|
78
|
+
from .logging_events import events
|
|
79
|
+
|
|
80
|
+
log = get_logger(__name__)
|
|
81
|
+
|
|
82
|
+
log.event(events.pallet_recorded, f"Pallet recorded: {store}", {"store": store, "pallets": n})
|
|
83
|
+
log.info("Report email queued") # category system
|
|
84
|
+
log.error("CI query failed", error=exc, data={"store": "CAM"})
|
|
85
|
+
log.event(events.weekly_report_sent, "...", data, level="error", duration_ms=ms)
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Plain `logging.getLogger(...)` calls keep working and produce valid lines; any `extra=` values
|
|
89
|
+
become `data`. Names are snake_case; `run_started`, `run_finished`, `http_request` and
|
|
90
|
+
`log_files_deleted` are written by the package. Categories: `request`, `business`, `job`,
|
|
91
|
+
`system`.
|
|
92
|
+
|
|
93
|
+
## Jobs, commands and scripts
|
|
94
|
+
|
|
95
|
+
A fresh trace id per run, and `run_started` / `run_finished` with the result and duration. A
|
|
96
|
+
failure is logged at `error` and re-raised; `SystemExit(0)` counts as success.
|
|
97
|
+
|
|
98
|
+
```python
|
|
99
|
+
from autoshiploads_logging import run_job
|
|
100
|
+
|
|
101
|
+
with run_job(log, "weekly_report"):
|
|
102
|
+
...
|
|
103
|
+
|
|
104
|
+
@run_job(log, "nightly_import") # APScheduler job, management command
|
|
105
|
+
def nightly_import(): ...
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
A script outside an app calls `setup_logging("<script>", "<version>",
|
|
109
|
+
log_dir=r"C:\ProgramData\Shiploads\Logs\<script>")` first (spec section 5).
|
|
110
|
+
|
|
111
|
+
## A longer-kept stream
|
|
112
|
+
|
|
113
|
+
For events people ask about long after (COB's picks, kept a year): those loggers write to their
|
|
114
|
+
own file with their own retention, and not to the main file.
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
from autoshiploads_logging import StreamConfig, setup_logging
|
|
118
|
+
|
|
119
|
+
setup_logging(
|
|
120
|
+
"cob",
|
|
121
|
+
"2.4.0",
|
|
122
|
+
streams={"picks": StreamConfig(loggers=("stock_movement",), retention_days=364)},
|
|
123
|
+
)
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
## Options
|
|
127
|
+
|
|
128
|
+
| Option | Default |
|
|
129
|
+
|---|---|
|
|
130
|
+
| `service`, `version` | required |
|
|
131
|
+
| `level` | `LOG_LEVEL` env, else `info` |
|
|
132
|
+
| `log_dir` | `LOG_DIR` env, else `<cwd>/logs` |
|
|
133
|
+
| `retention_days` | `LOG_RETENTION_DAYS`, else 90 |
|
|
134
|
+
| `time_zone` | `LOG_TIME_ZONE`, else Hobart |
|
|
135
|
+
| `output` | `file`; `stdout` for containers |
|
|
136
|
+
| `console` | `LOG_CONSOLE` 1/0, else terminal only |
|
|
137
|
+
| `redact_keys` | none (adds to the spec's list) |
|
|
138
|
+
| `streams` | none |
|
|
139
|
+
| `library_categories` | adds to the built-in map |
|
|
140
|
+
|
|
141
|
+
`LOG_LEVEL` wins over `level`, so a host can turn on `debug` without a code change. A bad value
|
|
142
|
+
from the environment is reported once as a `warn` line at startup and the default used.
|
|
143
|
+
|
|
144
|
+
## Files, retention, safety
|
|
145
|
+
|
|
146
|
+
- `logs/<service>_<YYYY-MM-DD>.jsonl`, dated by the Monday the week starts in Hobart; each line
|
|
147
|
+
picks its file from its own time, so rotation needs no timer.
|
|
148
|
+
- One write per line on an append-only handle, so several processes (waitress, a management
|
|
149
|
+
command, the dev reloader) can share a file without mixing lines. On Windows this opens the
|
|
150
|
+
file with `FILE_APPEND_DATA` directly: Python's own append mode goes through the C runtime,
|
|
151
|
+
which seeks and then writes in two steps, and in testing three processes writing one file
|
|
152
|
+
that way produced broken lines.
|
|
153
|
+
- Retention runs at setup and every 24 hours, by the date in the file name only: a weekly file
|
|
154
|
+
goes once its Sunday is past retention; WinSW's daily `*.yyyyMMdd.out.log` / `.err.log` once
|
|
155
|
+
their day is; a stream's files by the stream's own retention. Nothing else is touched.
|
|
156
|
+
- A failed write is never raised into the app; the first one prints a line to stderr.
|
|
157
|
+
- Keys in the spec's never-log list (`password`, `token`, `authorization`, `cookie`, ...) are
|
|
158
|
+
dropped from `data` at any depth. Objects JSON can't hold are written as their type name
|
|
159
|
+
only, never their repr (Django's request repr includes the query string).
|
|
160
|
+
|
|
161
|
+
## Changing this package
|
|
162
|
+
|
|
163
|
+
The spec (`~/.claude/logging-spec.md`) changes first, then this package (spec section 9).
|
|
164
|
+
Adding a field or option: minor version. Renaming, removing or retyping a field: major version.
|
|
165
|
+
See the repo README for releasing.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "autoshiploads-logging"
|
|
3
|
+
version = "1.0.0"
|
|
4
|
+
description = "JSONL logging for Shiploads Python apps: the logging spec's line (ECS names), weekly files named by the Monday in Hobart, retention, event catalogue, trace ids, Django request logging."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.11"
|
|
7
|
+
license = "LicenseRef-Proprietary"
|
|
8
|
+
keywords = ["logging", "jsonl", "ecs", "django"]
|
|
9
|
+
classifiers = [
|
|
10
|
+
"Programming Language :: Python :: 3",
|
|
11
|
+
"Framework :: Django",
|
|
12
|
+
"Operating System :: Microsoft :: Windows",
|
|
13
|
+
"Operating System :: POSIX",
|
|
14
|
+
]
|
|
15
|
+
# zoneinfo needs the IANA time zone data, which Windows doesn't ship. Django already depends
|
|
16
|
+
# on tzdata on Windows; it's declared here so scripts without Django get it too.
|
|
17
|
+
dependencies = ["tzdata; sys_platform == 'win32'"]
|
|
18
|
+
|
|
19
|
+
[dependency-groups]
|
|
20
|
+
dev = ["pytest==9.2.0", "django==5.2.19"]
|
|
21
|
+
|
|
22
|
+
[build-system]
|
|
23
|
+
requires = ["uv_build>=0.11.15,<0.12"]
|
|
24
|
+
build-backend = "uv_build"
|
|
25
|
+
|
|
26
|
+
[tool.pytest.ini_options]
|
|
27
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""autoshiploads-logging: the Shiploads logging spec (~/.claude/logging-spec.md) for Python.
|
|
2
|
+
|
|
3
|
+
from autoshiploads_logging import setup_logging, get_logger, define_events, run_job
|
|
4
|
+
|
|
5
|
+
setup_logging("cob", "2.4.0")
|
|
6
|
+
log = get_logger(__name__)
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from ._events import CATEGORIES, RESERVED_ACTIONS, EventSpec, Events, define_events
|
|
10
|
+
from ._format import DEFAULT_LIBRARY_CATEGORIES, JsonLineFormatter
|
|
11
|
+
from ._handlers import WeeklyFileHandler
|
|
12
|
+
from ._job import run_job
|
|
13
|
+
from ._logger import AppLogger, get_logger
|
|
14
|
+
from ._retention import SweepResult, sweep_expired
|
|
15
|
+
from ._sanitize import SPEC_SECRET_KEYS, serialize_error
|
|
16
|
+
from ._setup import DEFAULT_RETENTION_DAYS, LoggingSetup, StreamConfig, setup_logging
|
|
17
|
+
from ._time import DEFAULT_TIME_ZONE, week_start
|
|
18
|
+
from ._trace import current_trace_id, is_sane_trace_id, new_trace_id, with_trace
|
|
19
|
+
from ._version import LOGGER_ID, __version__
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"CATEGORIES",
|
|
23
|
+
"DEFAULT_LIBRARY_CATEGORIES",
|
|
24
|
+
"DEFAULT_RETENTION_DAYS",
|
|
25
|
+
"DEFAULT_TIME_ZONE",
|
|
26
|
+
"LOGGER_ID",
|
|
27
|
+
"RESERVED_ACTIONS",
|
|
28
|
+
"SPEC_SECRET_KEYS",
|
|
29
|
+
"AppLogger",
|
|
30
|
+
"EventSpec",
|
|
31
|
+
"Events",
|
|
32
|
+
"JsonLineFormatter",
|
|
33
|
+
"LoggingSetup",
|
|
34
|
+
"StreamConfig",
|
|
35
|
+
"SweepResult",
|
|
36
|
+
"WeeklyFileHandler",
|
|
37
|
+
"__version__",
|
|
38
|
+
"current_trace_id",
|
|
39
|
+
"define_events",
|
|
40
|
+
"get_logger",
|
|
41
|
+
"is_sane_trace_id",
|
|
42
|
+
"new_trace_id",
|
|
43
|
+
"run_job",
|
|
44
|
+
"serialize_error",
|
|
45
|
+
"setup_logging",
|
|
46
|
+
"sweep_expired",
|
|
47
|
+
"week_start",
|
|
48
|
+
"with_trace",
|
|
49
|
+
]
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""The app's event catalogue (spec section 3): every business action is a named event, declared
|
|
2
|
+
once in the app (for example `<app>/logging_events.py`):
|
|
3
|
+
|
|
4
|
+
events = define_events(
|
|
5
|
+
split_processed="business",
|
|
6
|
+
picks_confirmed="business",
|
|
7
|
+
)
|
|
8
|
+
log.event(events.split_processed, "Split processed: 243 products", {"products": 243})
|
|
9
|
+
|
|
10
|
+
Names are checked when the catalogue is defined, so a bad one fails at import, never at a log call.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import re
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from types import SimpleNamespace
|
|
18
|
+
from typing import Literal
|
|
19
|
+
|
|
20
|
+
Category = Literal["request", "business", "job", "system"]
|
|
21
|
+
CATEGORIES: tuple[str, ...] = ("request", "business", "job", "system")
|
|
22
|
+
|
|
23
|
+
#: Actions this package writes itself; an app's catalogue can't reuse them.
|
|
24
|
+
RESERVED_ACTIONS: tuple[str, ...] = ("run_started", "run_finished", "http_request", "log_files_deleted")
|
|
25
|
+
|
|
26
|
+
_SNAKE = re.compile(r"^[a-z][a-z0-9]*(?:_[a-z0-9]+)*$")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class EventSpec:
|
|
31
|
+
"""A catalogue entry: `event.action` and `event.category`."""
|
|
32
|
+
|
|
33
|
+
action: str
|
|
34
|
+
category: str
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class Events(SimpleNamespace):
|
|
38
|
+
"""The catalogue: one attribute per event, each an EventSpec."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def define_events(**events: str) -> Events:
|
|
42
|
+
"""`define_events(name="business", ...)`: the category of each named event."""
|
|
43
|
+
specs = {}
|
|
44
|
+
for action, category in events.items():
|
|
45
|
+
if not _SNAKE.match(action):
|
|
46
|
+
raise ValueError(f'Event name "{action}" must be snake_case (spec section 3)')
|
|
47
|
+
if action in RESERVED_ACTIONS:
|
|
48
|
+
raise ValueError(f'Event name "{action}" is written by autoshiploads-logging itself; choose another')
|
|
49
|
+
if category not in CATEGORIES:
|
|
50
|
+
raise ValueError(f'Event "{action}" has category "{category}"; use one of {", ".join(CATEGORIES)}')
|
|
51
|
+
specs[action] = EventSpec(action=action, category=category)
|
|
52
|
+
return Events(**specs)
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
"""Turns a stdlib LogRecord into the spec's line (section 2): nested ECS objects, in the spec's
|
|
2
|
+
order, absent fields left out (never null).
|
|
3
|
+
|
|
4
|
+
Where each field comes from:
|
|
5
|
+
- message, level, time, exception: the record itself, so plain `logging.getLogger(...).info()`
|
|
6
|
+
calls from the app, Django, waitress or APScheduler all produce valid lines.
|
|
7
|
+
- category, action, data, duration, http: the `_sl` dict that `AppLogger` (get_logger) attaches;
|
|
8
|
+
without it, category comes from the logger's name (`library_categories`), and any other
|
|
9
|
+
`extra=` values become `data`.
|
|
10
|
+
- trace.id: the current context's id (with_trace / run_job / the Django middleware)."""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import logging
|
|
16
|
+
from collections.abc import Callable, Mapping
|
|
17
|
+
from typing import Any
|
|
18
|
+
from zoneinfo import ZoneInfo
|
|
19
|
+
|
|
20
|
+
from ._events import CATEGORIES
|
|
21
|
+
from ._sanitize import serialize_error
|
|
22
|
+
from ._time import clock_time, iso_utc, utc_from_timestamp
|
|
23
|
+
from ._trace import current_trace_id
|
|
24
|
+
from ._version import LOGGER_ID
|
|
25
|
+
|
|
26
|
+
#: Category for lines from libraries, by logger name (the longest matching dotted prefix wins).
|
|
27
|
+
DEFAULT_LIBRARY_CATEGORIES: dict[str, str] = {
|
|
28
|
+
"django.request": "request",
|
|
29
|
+
"django.server": "request",
|
|
30
|
+
"django.db": "system",
|
|
31
|
+
"waitress": "system",
|
|
32
|
+
"apscheduler": "job",
|
|
33
|
+
"django_apscheduler": "job",
|
|
34
|
+
"uvicorn.access": "request",
|
|
35
|
+
"uvicorn": "system",
|
|
36
|
+
"py.warnings": "system",
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
_STANDARD_ATTRS = set(logging.LogRecord("x", 0, "x", 0, "x", None, None).__dict__) | {"message", "asctime", "taskName"}
|
|
40
|
+
# Extras Django attaches to django.request / django.server records: read for the http fields,
|
|
41
|
+
# never written as data (the request object's repr includes the query string).
|
|
42
|
+
_DJANGO_EXTRAS = {"request", "status_code", "server_time"}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def level_name(levelno: int) -> str:
|
|
46
|
+
if levelno >= logging.CRITICAL:
|
|
47
|
+
return "fatal"
|
|
48
|
+
if levelno >= logging.ERROR:
|
|
49
|
+
return "error"
|
|
50
|
+
if levelno >= logging.WARNING:
|
|
51
|
+
return "warn"
|
|
52
|
+
if levelno >= logging.INFO:
|
|
53
|
+
return "info"
|
|
54
|
+
return "debug"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class JsonLineFormatter(logging.Formatter):
|
|
58
|
+
def __init__(
|
|
59
|
+
self,
|
|
60
|
+
service: str,
|
|
61
|
+
version: str,
|
|
62
|
+
host: str,
|
|
63
|
+
sanitize: Callable[[Any], Any],
|
|
64
|
+
library_categories: Mapping[str, str] | None = None,
|
|
65
|
+
) -> None:
|
|
66
|
+
super().__init__()
|
|
67
|
+
self.service = service
|
|
68
|
+
self.version = version
|
|
69
|
+
self.host = host
|
|
70
|
+
self.sanitize = sanitize
|
|
71
|
+
cats = {**DEFAULT_LIBRARY_CATEGORIES, **(library_categories or {})}
|
|
72
|
+
# longest prefix first
|
|
73
|
+
self._library = sorted(cats.items(), key=lambda kv: -len(kv[0]))
|
|
74
|
+
|
|
75
|
+
def _category_for(self, logger_name: str) -> str:
|
|
76
|
+
for prefix, cat in self._library:
|
|
77
|
+
if logger_name == prefix or logger_name.startswith(prefix + "."):
|
|
78
|
+
return cat
|
|
79
|
+
return "system"
|
|
80
|
+
|
|
81
|
+
def build(self, record: logging.LogRecord) -> dict[str, Any]:
|
|
82
|
+
sl: dict[str, Any] = getattr(record, "_sl", None) or {}
|
|
83
|
+
try:
|
|
84
|
+
message = record.getMessage()
|
|
85
|
+
except Exception:
|
|
86
|
+
message = f"{record.msg} (log arguments could not be formatted)"
|
|
87
|
+
|
|
88
|
+
line: dict[str, Any] = {
|
|
89
|
+
"@timestamp": iso_utc(utc_from_timestamp(record.created)),
|
|
90
|
+
"log": {"level": level_name(record.levelno), "logger": LOGGER_ID},
|
|
91
|
+
"message": str(message),
|
|
92
|
+
"service": {"name": self.service, "version": self.version},
|
|
93
|
+
"host": {"name": self.host},
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
category = sl.get("category")
|
|
97
|
+
if category not in CATEGORIES:
|
|
98
|
+
category = self._category_for(record.name)
|
|
99
|
+
event: dict[str, Any] = {"category": category}
|
|
100
|
+
if sl.get("action"):
|
|
101
|
+
event["action"] = sl["action"]
|
|
102
|
+
duration = sl.get("duration_ms")
|
|
103
|
+
if isinstance(duration, (int, float)) and duration == duration:
|
|
104
|
+
event["duration_ms"] = round(duration)
|
|
105
|
+
line["event"] = event
|
|
106
|
+
|
|
107
|
+
trace_id = sl.get("trace_id") or current_trace_id()
|
|
108
|
+
if trace_id:
|
|
109
|
+
line["trace"] = {"id": trace_id}
|
|
110
|
+
|
|
111
|
+
http = dict(sl.get("http") or {})
|
|
112
|
+
req = record.__dict__.get("request")
|
|
113
|
+
if req is not None and not http:
|
|
114
|
+
method, path = getattr(req, "method", None), getattr(req, "path", None)
|
|
115
|
+
if isinstance(method, str):
|
|
116
|
+
http["method"] = method
|
|
117
|
+
if isinstance(path, str):
|
|
118
|
+
http["path"] = path
|
|
119
|
+
status = record.__dict__.get("status_code")
|
|
120
|
+
if isinstance(status, int) and "status" not in http:
|
|
121
|
+
http["status"] = status
|
|
122
|
+
if http:
|
|
123
|
+
h: dict[str, Any] = {}
|
|
124
|
+
if http.get("method"):
|
|
125
|
+
h["request"] = {"method": http["method"]}
|
|
126
|
+
if isinstance(http.get("status"), int):
|
|
127
|
+
h["response"] = {"status_code": http["status"]}
|
|
128
|
+
if h:
|
|
129
|
+
line["http"] = h
|
|
130
|
+
if http.get("path"):
|
|
131
|
+
line["url"] = {"path": http["path"]}
|
|
132
|
+
if http.get("client_ip"):
|
|
133
|
+
line["client"] = {"ip": http["client_ip"]}
|
|
134
|
+
|
|
135
|
+
if record.exc_info and record.exc_info[1] is not None:
|
|
136
|
+
line["error"] = serialize_error(record.exc_info[1])
|
|
137
|
+
elif "error_value" in sl:
|
|
138
|
+
line["error"] = {"type": "NonError", "message": str(sl["error_value"])}
|
|
139
|
+
|
|
140
|
+
data: dict[str, Any] = {}
|
|
141
|
+
if isinstance(sl.get("data"), Mapping):
|
|
142
|
+
data.update(sl["data"])
|
|
143
|
+
elif sl.get("data") is not None:
|
|
144
|
+
data["value"] = sl["data"]
|
|
145
|
+
for key, value in record.__dict__.items():
|
|
146
|
+
if key in _STANDARD_ATTRS or key in _DJANGO_EXTRAS or key.startswith("_"):
|
|
147
|
+
continue
|
|
148
|
+
data.setdefault(key, value)
|
|
149
|
+
if data:
|
|
150
|
+
clean = self.sanitize(data)
|
|
151
|
+
if clean:
|
|
152
|
+
line["data"] = clean
|
|
153
|
+
return line
|
|
154
|
+
|
|
155
|
+
def format(self, record: logging.LogRecord) -> str:
|
|
156
|
+
line = self.build(record)
|
|
157
|
+
try:
|
|
158
|
+
return json.dumps(line, ensure_ascii=False, separators=(",", ":"))
|
|
159
|
+
except Exception as exc: # sanitising makes this unreachable; the line must still be written
|
|
160
|
+
line.pop("data", None)
|
|
161
|
+
line["error"] = serialize_error(exc)
|
|
162
|
+
line["message"] = f"{line['message']} (log line could not be serialised)"
|
|
163
|
+
return json.dumps(line, ensure_ascii=False, separators=(",", ":"), default=str)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
_COLOURS = {"debug": "\x1b[90m", "info": "\x1b[36m", "warn": "\x1b[33m", "error": "\x1b[31m", "fatal": "\x1b[35m"}
|
|
167
|
+
_RESET = "\x1b[0m"
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class ConsoleFormatter(logging.Formatter):
|
|
171
|
+
"""The readable development line: local time, level, app, message, data, then the stack."""
|
|
172
|
+
|
|
173
|
+
def __init__(self, json_formatter: JsonLineFormatter, tz: ZoneInfo, colour: bool) -> None:
|
|
174
|
+
super().__init__()
|
|
175
|
+
self.json_formatter = json_formatter
|
|
176
|
+
self.tz = tz
|
|
177
|
+
self.colour = colour
|
|
178
|
+
|
|
179
|
+
def format(self, record: logging.LogRecord) -> str:
|
|
180
|
+
line = self.json_formatter.build(record)
|
|
181
|
+
level = line["log"]["level"]
|
|
182
|
+
label = level.upper().ljust(5)
|
|
183
|
+
if self.colour:
|
|
184
|
+
label = _COLOURS[level] + label + _RESET
|
|
185
|
+
out = f"{clock_time(utc_from_timestamp(record.created), self.tz)} {label} {self.json_formatter.service} {line['message']}"
|
|
186
|
+
if "data" in line:
|
|
187
|
+
try:
|
|
188
|
+
out += " " + json.dumps(line["data"], ensure_ascii=False, separators=(",", ":"))
|
|
189
|
+
except Exception:
|
|
190
|
+
pass
|
|
191
|
+
err = line.get("error")
|
|
192
|
+
if err:
|
|
193
|
+
out += "\n" + err.get("stack_trace", f"{err['type']}: {err['message']}")
|
|
194
|
+
return out
|