autoshiploads-logging 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- autoshiploads_logging/__init__.py +49 -0
- autoshiploads_logging/_events.py +52 -0
- autoshiploads_logging/_format.py +194 -0
- autoshiploads_logging/_handlers.py +73 -0
- autoshiploads_logging/_job.py +61 -0
- autoshiploads_logging/_logger.py +113 -0
- autoshiploads_logging/_retention.py +80 -0
- autoshiploads_logging/_sanitize.py +101 -0
- autoshiploads_logging/_setup.py +282 -0
- autoshiploads_logging/_time.py +42 -0
- autoshiploads_logging/_trace.py +46 -0
- autoshiploads_logging/_version.py +11 -0
- autoshiploads_logging/_writer.py +84 -0
- autoshiploads_logging/django.py +70 -0
- autoshiploads_logging/py.typed +0 -0
- autoshiploads_logging-1.0.0.dist-info/METADATA +179 -0
- autoshiploads_logging-1.0.0.dist-info/RECORD +18 -0
- autoshiploads_logging-1.0.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""autoshiploads-logging: the Shiploads logging spec (~/.claude/logging-spec.md) for Python.
|
|
2
|
+
|
|
3
|
+
from autoshiploads_logging import setup_logging, get_logger, define_events, run_job
|
|
4
|
+
|
|
5
|
+
setup_logging("cob", "2.4.0")
|
|
6
|
+
log = get_logger(__name__)
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from ._events import CATEGORIES, RESERVED_ACTIONS, EventSpec, Events, define_events
|
|
10
|
+
from ._format import DEFAULT_LIBRARY_CATEGORIES, JsonLineFormatter
|
|
11
|
+
from ._handlers import WeeklyFileHandler
|
|
12
|
+
from ._job import run_job
|
|
13
|
+
from ._logger import AppLogger, get_logger
|
|
14
|
+
from ._retention import SweepResult, sweep_expired
|
|
15
|
+
from ._sanitize import SPEC_SECRET_KEYS, serialize_error
|
|
16
|
+
from ._setup import DEFAULT_RETENTION_DAYS, LoggingSetup, StreamConfig, setup_logging
|
|
17
|
+
from ._time import DEFAULT_TIME_ZONE, week_start
|
|
18
|
+
from ._trace import current_trace_id, is_sane_trace_id, new_trace_id, with_trace
|
|
19
|
+
from ._version import LOGGER_ID, __version__
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"CATEGORIES",
|
|
23
|
+
"DEFAULT_LIBRARY_CATEGORIES",
|
|
24
|
+
"DEFAULT_RETENTION_DAYS",
|
|
25
|
+
"DEFAULT_TIME_ZONE",
|
|
26
|
+
"LOGGER_ID",
|
|
27
|
+
"RESERVED_ACTIONS",
|
|
28
|
+
"SPEC_SECRET_KEYS",
|
|
29
|
+
"AppLogger",
|
|
30
|
+
"EventSpec",
|
|
31
|
+
"Events",
|
|
32
|
+
"JsonLineFormatter",
|
|
33
|
+
"LoggingSetup",
|
|
34
|
+
"StreamConfig",
|
|
35
|
+
"SweepResult",
|
|
36
|
+
"WeeklyFileHandler",
|
|
37
|
+
"__version__",
|
|
38
|
+
"current_trace_id",
|
|
39
|
+
"define_events",
|
|
40
|
+
"get_logger",
|
|
41
|
+
"is_sane_trace_id",
|
|
42
|
+
"new_trace_id",
|
|
43
|
+
"run_job",
|
|
44
|
+
"serialize_error",
|
|
45
|
+
"setup_logging",
|
|
46
|
+
"sweep_expired",
|
|
47
|
+
"week_start",
|
|
48
|
+
"with_trace",
|
|
49
|
+
]
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""The app's event catalogue (spec section 3): every business action is a named event, declared
|
|
2
|
+
once in the app (for example `<app>/logging_events.py`):
|
|
3
|
+
|
|
4
|
+
events = define_events(
|
|
5
|
+
split_processed="business",
|
|
6
|
+
picks_confirmed="business",
|
|
7
|
+
)
|
|
8
|
+
log.event(events.split_processed, "Split processed: 243 products", {"products": 243})
|
|
9
|
+
|
|
10
|
+
Names are checked when the catalogue is defined, so a bad one fails at import, never at a log call.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import re
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from types import SimpleNamespace
|
|
18
|
+
from typing import Literal
|
|
19
|
+
|
|
20
|
+
Category = Literal["request", "business", "job", "system"]
|
|
21
|
+
CATEGORIES: tuple[str, ...] = ("request", "business", "job", "system")
|
|
22
|
+
|
|
23
|
+
#: Actions this package writes itself; an app's catalogue can't reuse them.
|
|
24
|
+
RESERVED_ACTIONS: tuple[str, ...] = ("run_started", "run_finished", "http_request", "log_files_deleted")
|
|
25
|
+
|
|
26
|
+
_SNAKE = re.compile(r"^[a-z][a-z0-9]*(?:_[a-z0-9]+)*$")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class EventSpec:
|
|
31
|
+
"""A catalogue entry: `event.action` and `event.category`."""
|
|
32
|
+
|
|
33
|
+
action: str
|
|
34
|
+
category: str
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class Events(SimpleNamespace):
|
|
38
|
+
"""The catalogue: one attribute per event, each an EventSpec."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def define_events(**events: str) -> Events:
|
|
42
|
+
"""`define_events(name="business", ...)`: the category of each named event."""
|
|
43
|
+
specs = {}
|
|
44
|
+
for action, category in events.items():
|
|
45
|
+
if not _SNAKE.match(action):
|
|
46
|
+
raise ValueError(f'Event name "{action}" must be snake_case (spec section 3)')
|
|
47
|
+
if action in RESERVED_ACTIONS:
|
|
48
|
+
raise ValueError(f'Event name "{action}" is written by autoshiploads-logging itself; choose another')
|
|
49
|
+
if category not in CATEGORIES:
|
|
50
|
+
raise ValueError(f'Event "{action}" has category "{category}"; use one of {", ".join(CATEGORIES)}')
|
|
51
|
+
specs[action] = EventSpec(action=action, category=category)
|
|
52
|
+
return Events(**specs)
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
"""Turns a stdlib LogRecord into the spec's line (section 2): nested ECS objects, in the spec's
|
|
2
|
+
order, absent fields left out (never null).
|
|
3
|
+
|
|
4
|
+
Where each field comes from:
|
|
5
|
+
- message, level, time, exception: the record itself, so plain `logging.getLogger(...).info()`
|
|
6
|
+
calls from the app, Django, waitress or APScheduler all produce valid lines.
|
|
7
|
+
- category, action, data, duration, http: the `_sl` dict that `AppLogger` (get_logger) attaches;
|
|
8
|
+
without it, category comes from the logger's name (`library_categories`), and any other
|
|
9
|
+
`extra=` values become `data`.
|
|
10
|
+
- trace.id: the current context's id (with_trace / run_job / the Django middleware)."""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import logging
|
|
16
|
+
from collections.abc import Callable, Mapping
|
|
17
|
+
from typing import Any
|
|
18
|
+
from zoneinfo import ZoneInfo
|
|
19
|
+
|
|
20
|
+
from ._events import CATEGORIES
|
|
21
|
+
from ._sanitize import serialize_error
|
|
22
|
+
from ._time import clock_time, iso_utc, utc_from_timestamp
|
|
23
|
+
from ._trace import current_trace_id
|
|
24
|
+
from ._version import LOGGER_ID
|
|
25
|
+
|
|
26
|
+
#: Category for lines from libraries, by logger name (the longest matching dotted prefix wins).
|
|
27
|
+
DEFAULT_LIBRARY_CATEGORIES: dict[str, str] = {
|
|
28
|
+
"django.request": "request",
|
|
29
|
+
"django.server": "request",
|
|
30
|
+
"django.db": "system",
|
|
31
|
+
"waitress": "system",
|
|
32
|
+
"apscheduler": "job",
|
|
33
|
+
"django_apscheduler": "job",
|
|
34
|
+
"uvicorn.access": "request",
|
|
35
|
+
"uvicorn": "system",
|
|
36
|
+
"py.warnings": "system",
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
_STANDARD_ATTRS = set(logging.LogRecord("x", 0, "x", 0, "x", None, None).__dict__) | {"message", "asctime", "taskName"}
|
|
40
|
+
# Extras Django attaches to django.request / django.server records: read for the http fields,
|
|
41
|
+
# never written as data (the request object's repr includes the query string).
|
|
42
|
+
_DJANGO_EXTRAS = {"request", "status_code", "server_time"}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def level_name(levelno: int) -> str:
|
|
46
|
+
if levelno >= logging.CRITICAL:
|
|
47
|
+
return "fatal"
|
|
48
|
+
if levelno >= logging.ERROR:
|
|
49
|
+
return "error"
|
|
50
|
+
if levelno >= logging.WARNING:
|
|
51
|
+
return "warn"
|
|
52
|
+
if levelno >= logging.INFO:
|
|
53
|
+
return "info"
|
|
54
|
+
return "debug"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class JsonLineFormatter(logging.Formatter):
|
|
58
|
+
def __init__(
|
|
59
|
+
self,
|
|
60
|
+
service: str,
|
|
61
|
+
version: str,
|
|
62
|
+
host: str,
|
|
63
|
+
sanitize: Callable[[Any], Any],
|
|
64
|
+
library_categories: Mapping[str, str] | None = None,
|
|
65
|
+
) -> None:
|
|
66
|
+
super().__init__()
|
|
67
|
+
self.service = service
|
|
68
|
+
self.version = version
|
|
69
|
+
self.host = host
|
|
70
|
+
self.sanitize = sanitize
|
|
71
|
+
cats = {**DEFAULT_LIBRARY_CATEGORIES, **(library_categories or {})}
|
|
72
|
+
# longest prefix first
|
|
73
|
+
self._library = sorted(cats.items(), key=lambda kv: -len(kv[0]))
|
|
74
|
+
|
|
75
|
+
def _category_for(self, logger_name: str) -> str:
|
|
76
|
+
for prefix, cat in self._library:
|
|
77
|
+
if logger_name == prefix or logger_name.startswith(prefix + "."):
|
|
78
|
+
return cat
|
|
79
|
+
return "system"
|
|
80
|
+
|
|
81
|
+
def build(self, record: logging.LogRecord) -> dict[str, Any]:
|
|
82
|
+
sl: dict[str, Any] = getattr(record, "_sl", None) or {}
|
|
83
|
+
try:
|
|
84
|
+
message = record.getMessage()
|
|
85
|
+
except Exception:
|
|
86
|
+
message = f"{record.msg} (log arguments could not be formatted)"
|
|
87
|
+
|
|
88
|
+
line: dict[str, Any] = {
|
|
89
|
+
"@timestamp": iso_utc(utc_from_timestamp(record.created)),
|
|
90
|
+
"log": {"level": level_name(record.levelno), "logger": LOGGER_ID},
|
|
91
|
+
"message": str(message),
|
|
92
|
+
"service": {"name": self.service, "version": self.version},
|
|
93
|
+
"host": {"name": self.host},
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
category = sl.get("category")
|
|
97
|
+
if category not in CATEGORIES:
|
|
98
|
+
category = self._category_for(record.name)
|
|
99
|
+
event: dict[str, Any] = {"category": category}
|
|
100
|
+
if sl.get("action"):
|
|
101
|
+
event["action"] = sl["action"]
|
|
102
|
+
duration = sl.get("duration_ms")
|
|
103
|
+
if isinstance(duration, (int, float)) and duration == duration:
|
|
104
|
+
event["duration_ms"] = round(duration)
|
|
105
|
+
line["event"] = event
|
|
106
|
+
|
|
107
|
+
trace_id = sl.get("trace_id") or current_trace_id()
|
|
108
|
+
if trace_id:
|
|
109
|
+
line["trace"] = {"id": trace_id}
|
|
110
|
+
|
|
111
|
+
http = dict(sl.get("http") or {})
|
|
112
|
+
req = record.__dict__.get("request")
|
|
113
|
+
if req is not None and not http:
|
|
114
|
+
method, path = getattr(req, "method", None), getattr(req, "path", None)
|
|
115
|
+
if isinstance(method, str):
|
|
116
|
+
http["method"] = method
|
|
117
|
+
if isinstance(path, str):
|
|
118
|
+
http["path"] = path
|
|
119
|
+
status = record.__dict__.get("status_code")
|
|
120
|
+
if isinstance(status, int) and "status" not in http:
|
|
121
|
+
http["status"] = status
|
|
122
|
+
if http:
|
|
123
|
+
h: dict[str, Any] = {}
|
|
124
|
+
if http.get("method"):
|
|
125
|
+
h["request"] = {"method": http["method"]}
|
|
126
|
+
if isinstance(http.get("status"), int):
|
|
127
|
+
h["response"] = {"status_code": http["status"]}
|
|
128
|
+
if h:
|
|
129
|
+
line["http"] = h
|
|
130
|
+
if http.get("path"):
|
|
131
|
+
line["url"] = {"path": http["path"]}
|
|
132
|
+
if http.get("client_ip"):
|
|
133
|
+
line["client"] = {"ip": http["client_ip"]}
|
|
134
|
+
|
|
135
|
+
if record.exc_info and record.exc_info[1] is not None:
|
|
136
|
+
line["error"] = serialize_error(record.exc_info[1])
|
|
137
|
+
elif "error_value" in sl:
|
|
138
|
+
line["error"] = {"type": "NonError", "message": str(sl["error_value"])}
|
|
139
|
+
|
|
140
|
+
data: dict[str, Any] = {}
|
|
141
|
+
if isinstance(sl.get("data"), Mapping):
|
|
142
|
+
data.update(sl["data"])
|
|
143
|
+
elif sl.get("data") is not None:
|
|
144
|
+
data["value"] = sl["data"]
|
|
145
|
+
for key, value in record.__dict__.items():
|
|
146
|
+
if key in _STANDARD_ATTRS or key in _DJANGO_EXTRAS or key.startswith("_"):
|
|
147
|
+
continue
|
|
148
|
+
data.setdefault(key, value)
|
|
149
|
+
if data:
|
|
150
|
+
clean = self.sanitize(data)
|
|
151
|
+
if clean:
|
|
152
|
+
line["data"] = clean
|
|
153
|
+
return line
|
|
154
|
+
|
|
155
|
+
def format(self, record: logging.LogRecord) -> str:
|
|
156
|
+
line = self.build(record)
|
|
157
|
+
try:
|
|
158
|
+
return json.dumps(line, ensure_ascii=False, separators=(",", ":"))
|
|
159
|
+
except Exception as exc: # sanitising makes this unreachable; the line must still be written
|
|
160
|
+
line.pop("data", None)
|
|
161
|
+
line["error"] = serialize_error(exc)
|
|
162
|
+
line["message"] = f"{line['message']} (log line could not be serialised)"
|
|
163
|
+
return json.dumps(line, ensure_ascii=False, separators=(",", ":"), default=str)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
_COLOURS = {"debug": "\x1b[90m", "info": "\x1b[36m", "warn": "\x1b[33m", "error": "\x1b[31m", "fatal": "\x1b[35m"}
|
|
167
|
+
_RESET = "\x1b[0m"
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class ConsoleFormatter(logging.Formatter):
|
|
171
|
+
"""The readable development line: local time, level, app, message, data, then the stack."""
|
|
172
|
+
|
|
173
|
+
def __init__(self, json_formatter: JsonLineFormatter, tz: ZoneInfo, colour: bool) -> None:
|
|
174
|
+
super().__init__()
|
|
175
|
+
self.json_formatter = json_formatter
|
|
176
|
+
self.tz = tz
|
|
177
|
+
self.colour = colour
|
|
178
|
+
|
|
179
|
+
def format(self, record: logging.LogRecord) -> str:
|
|
180
|
+
line = self.json_formatter.build(record)
|
|
181
|
+
level = line["log"]["level"]
|
|
182
|
+
label = level.upper().ljust(5)
|
|
183
|
+
if self.colour:
|
|
184
|
+
label = _COLOURS[level] + label + _RESET
|
|
185
|
+
out = f"{clock_time(utc_from_timestamp(record.created), self.tz)} {label} {self.json_formatter.service} {line['message']}"
|
|
186
|
+
if "data" in line:
|
|
187
|
+
try:
|
|
188
|
+
out += " " + json.dumps(line["data"], ensure_ascii=False, separators=(",", ":"))
|
|
189
|
+
except Exception:
|
|
190
|
+
pass
|
|
191
|
+
err = line.get("error")
|
|
192
|
+
if err:
|
|
193
|
+
out += "\n" + err.get("stack_trace", f"{err['type']}: {err['message']}")
|
|
194
|
+
return out
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Handlers: the weekly file (spec section 5) and stdout for container hosts. Neither ever raises
|
|
2
|
+
into the app (spec principle 5); the first failed write prints one line to stderr."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import logging
|
|
7
|
+
import os
|
|
8
|
+
import sys
|
|
9
|
+
from zoneinfo import ZoneInfo
|
|
10
|
+
|
|
11
|
+
from ._time import utc_from_timestamp, week_start
|
|
12
|
+
from ._writer import AppendFile
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def weekly_file_name(prefix: str, created: float, tz: ZoneInfo) -> str:
|
|
16
|
+
return f"{prefix}_{week_start(utc_from_timestamp(created), tz)}.jsonl"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class WeeklyFileHandler(logging.Handler):
|
|
20
|
+
"""Writes each record as one line to `<directory>/<prefix>_<Monday>.jsonl`. The Monday comes
|
|
21
|
+
from the record's own time, so the week's file changes on its own at Monday 00:00 local time."""
|
|
22
|
+
|
|
23
|
+
def __init__(self, directory: str, prefix: str, tz: ZoneInfo, level: int = logging.NOTSET) -> None:
|
|
24
|
+
super().__init__(level)
|
|
25
|
+
self.directory = directory
|
|
26
|
+
self.prefix = prefix
|
|
27
|
+
self.tz = tz
|
|
28
|
+
self._file: AppendFile | None = None
|
|
29
|
+
self._path = ""
|
|
30
|
+
self._reported = False
|
|
31
|
+
|
|
32
|
+
def _close_file(self) -> None:
|
|
33
|
+
if self._file is not None:
|
|
34
|
+
try:
|
|
35
|
+
self._file.close()
|
|
36
|
+
except Exception:
|
|
37
|
+
pass
|
|
38
|
+
self._file = None
|
|
39
|
+
self._path = ""
|
|
40
|
+
|
|
41
|
+
def emit(self, record: logging.LogRecord) -> None:
|
|
42
|
+
try:
|
|
43
|
+
data = (self.format(record) + "\n").encode("utf-8")
|
|
44
|
+
path = os.path.join(self.directory, weekly_file_name(self.prefix, record.created, self.tz))
|
|
45
|
+
if path != self._path or self._file is None:
|
|
46
|
+
self._close_file()
|
|
47
|
+
os.makedirs(self.directory, exist_ok=True)
|
|
48
|
+
self._file = AppendFile(path)
|
|
49
|
+
self._path = path
|
|
50
|
+
self._file.write(data)
|
|
51
|
+
except Exception as exc:
|
|
52
|
+
self._close_file()
|
|
53
|
+
if not self._reported:
|
|
54
|
+
self._reported = True
|
|
55
|
+
try:
|
|
56
|
+
sys.stderr.write(f"[autoshiploads-logging] cannot write to {self.directory}: {exc}\n")
|
|
57
|
+
except Exception:
|
|
58
|
+
pass
|
|
59
|
+
|
|
60
|
+
def close(self) -> None:
|
|
61
|
+
self.acquire()
|
|
62
|
+
try:
|
|
63
|
+
self._close_file()
|
|
64
|
+
finally:
|
|
65
|
+
self.release()
|
|
66
|
+
super().close()
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class QuietStreamHandler(logging.StreamHandler):
|
|
70
|
+
"""A StreamHandler that swallows its own failures instead of printing a traceback per line."""
|
|
71
|
+
|
|
72
|
+
def handleError(self, record: logging.LogRecord) -> None: # noqa: N802 (stdlib name)
|
|
73
|
+
pass
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""One run of a scheduled job, management command or script (spec section 5): a fresh trace id,
|
|
2
|
+
`run_started`, then `run_finished` with the result and duration. A failure is logged at error
|
|
3
|
+
and re-raised, so the caller's own failure path (exit code, failure email) still runs.
|
|
4
|
+
|
|
5
|
+
with run_job(log, "daily_report"):
|
|
6
|
+
...
|
|
7
|
+
|
|
8
|
+
@run_job(log, "daily_report")
|
|
9
|
+
def daily_report(): ...
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import time
|
|
15
|
+
from collections.abc import Iterator
|
|
16
|
+
from contextlib import contextmanager
|
|
17
|
+
|
|
18
|
+
from ._logger import AppLogger
|
|
19
|
+
from ._trace import with_trace
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@contextmanager
|
|
23
|
+
def run_job(log: AppLogger, job: str, trace_id: str | None = None) -> Iterator[str]:
|
|
24
|
+
"""Yields the run's trace id. `trace_id` reuses an id a parent process passed on."""
|
|
25
|
+
with with_trace(trace_id) as tid:
|
|
26
|
+
started = time.perf_counter()
|
|
27
|
+
log.info(f"Run started: {job}", category="job", action="run_started", data={"job": job})
|
|
28
|
+
try:
|
|
29
|
+
yield tid
|
|
30
|
+
except SystemExit as exc:
|
|
31
|
+
if exc.code in (0, None):
|
|
32
|
+
_finished(log, job, started)
|
|
33
|
+
else:
|
|
34
|
+
_failed(log, job, started, exc)
|
|
35
|
+
raise
|
|
36
|
+
except BaseException as exc:
|
|
37
|
+
_failed(log, job, started, exc)
|
|
38
|
+
raise
|
|
39
|
+
else:
|
|
40
|
+
_finished(log, job, started)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _finished(log: AppLogger, job: str, started: float) -> None:
|
|
44
|
+
log.info(
|
|
45
|
+
f"Run finished: {job}",
|
|
46
|
+
category="job",
|
|
47
|
+
action="run_finished",
|
|
48
|
+
duration_ms=(time.perf_counter() - started) * 1000,
|
|
49
|
+
data={"job": job, "result": "success"},
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _failed(log: AppLogger, job: str, started: float, exc: BaseException) -> None:
|
|
54
|
+
log.error(
|
|
55
|
+
f"Run failed: {job}",
|
|
56
|
+
category="job",
|
|
57
|
+
action="run_finished",
|
|
58
|
+
duration_ms=(time.perf_counter() - started) * 1000,
|
|
59
|
+
data={"job": job, "result": "failure"},
|
|
60
|
+
error=exc,
|
|
61
|
+
)
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""`get_logger(name)`: a thin wrapper over a stdlib logger that adds the spec's fields (category,
|
|
2
|
+
action, data, error, duration) to the record. Lines still go through stdlib logging, so the
|
|
3
|
+
handlers `setup_logging` installed write them alongside every library's lines."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import logging
|
|
8
|
+
from collections.abc import Mapping
|
|
9
|
+
from typing import Any, Literal
|
|
10
|
+
|
|
11
|
+
from ._events import EventSpec
|
|
12
|
+
|
|
13
|
+
LevelName = Literal["debug", "info", "warn", "error", "fatal"]
|
|
14
|
+
|
|
15
|
+
LEVELS: dict[str, int] = {
|
|
16
|
+
"debug": logging.DEBUG,
|
|
17
|
+
"info": logging.INFO,
|
|
18
|
+
"warn": logging.WARNING,
|
|
19
|
+
"warning": logging.WARNING,
|
|
20
|
+
"error": logging.ERROR,
|
|
21
|
+
"fatal": logging.CRITICAL,
|
|
22
|
+
"critical": logging.CRITICAL,
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class AppLogger:
|
|
27
|
+
def __init__(self, name: str) -> None:
|
|
28
|
+
self.logger = logging.getLogger(name)
|
|
29
|
+
|
|
30
|
+
def log(
|
|
31
|
+
self,
|
|
32
|
+
level: LevelName | int,
|
|
33
|
+
message: str,
|
|
34
|
+
*,
|
|
35
|
+
category: str | None = None,
|
|
36
|
+
action: str | None = None,
|
|
37
|
+
data: Mapping[str, Any] | None = None,
|
|
38
|
+
error: BaseException | object | None = None,
|
|
39
|
+
duration_ms: float | None = None,
|
|
40
|
+
trace_id: str | None = None,
|
|
41
|
+
http: Mapping[str, Any] | None = None,
|
|
42
|
+
) -> None:
|
|
43
|
+
try:
|
|
44
|
+
levelno = level if isinstance(level, int) else LEVELS.get(level, logging.INFO)
|
|
45
|
+
if not self.logger.isEnabledFor(levelno):
|
|
46
|
+
return
|
|
47
|
+
sl: dict[str, Any] = {}
|
|
48
|
+
if category:
|
|
49
|
+
sl["category"] = category
|
|
50
|
+
if action:
|
|
51
|
+
sl["action"] = action
|
|
52
|
+
if data is not None:
|
|
53
|
+
sl["data"] = data
|
|
54
|
+
if duration_ms is not None:
|
|
55
|
+
sl["duration_ms"] = duration_ms
|
|
56
|
+
if trace_id:
|
|
57
|
+
sl["trace_id"] = trace_id
|
|
58
|
+
if http:
|
|
59
|
+
sl["http"] = dict(http)
|
|
60
|
+
exc_info = None
|
|
61
|
+
if isinstance(error, BaseException):
|
|
62
|
+
exc_info = (type(error), error, error.__traceback__)
|
|
63
|
+
elif error is not None:
|
|
64
|
+
sl["error_value"] = error
|
|
65
|
+
# stacklevel 3: report the app's call site, not this wrapper.
|
|
66
|
+
self.logger.log(levelno, message, exc_info=exc_info, extra={"_sl": sl}, stacklevel=3)
|
|
67
|
+
except Exception:
|
|
68
|
+
pass # spec principle 5
|
|
69
|
+
|
|
70
|
+
def debug(self, message: str, **fields: Any) -> None:
|
|
71
|
+
self.log("debug", message, **fields)
|
|
72
|
+
|
|
73
|
+
def info(self, message: str, **fields: Any) -> None:
|
|
74
|
+
self.log("info", message, **fields)
|
|
75
|
+
|
|
76
|
+
def warn(self, message: str, **fields: Any) -> None:
|
|
77
|
+
self.log("warn", message, **fields)
|
|
78
|
+
|
|
79
|
+
warning = warn
|
|
80
|
+
|
|
81
|
+
def error(self, message: str, **fields: Any) -> None:
|
|
82
|
+
self.log("error", message, **fields)
|
|
83
|
+
|
|
84
|
+
def fatal(self, message: str, **fields: Any) -> None:
|
|
85
|
+
self.log("fatal", message, **fields)
|
|
86
|
+
|
|
87
|
+
def event(
|
|
88
|
+
self,
|
|
89
|
+
spec: EventSpec,
|
|
90
|
+
message: str,
|
|
91
|
+
data: Mapping[str, Any] | None = None,
|
|
92
|
+
*,
|
|
93
|
+
level: LevelName = "info",
|
|
94
|
+
duration_ms: float | None = None,
|
|
95
|
+
error: BaseException | object | None = None,
|
|
96
|
+
trace_id: str | None = None,
|
|
97
|
+
) -> None:
|
|
98
|
+
"""Logs a catalogue event: category and action come from `spec`."""
|
|
99
|
+
self.log(
|
|
100
|
+
level,
|
|
101
|
+
message,
|
|
102
|
+
category=spec.category,
|
|
103
|
+
action=spec.action,
|
|
104
|
+
data=data,
|
|
105
|
+
duration_ms=duration_ms,
|
|
106
|
+
error=error,
|
|
107
|
+
trace_id=trace_id,
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def get_logger(name: str) -> AppLogger:
|
|
112
|
+
"""A logger for app code. Use the module's name (`get_logger(__name__)`) or the app's."""
|
|
113
|
+
return AppLogger(name)
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""Deletes expired log files, judged by the date in the file name, never the modified time
|
|
2
|
+
(spec section 5). Only these name patterns are touched:
|
|
3
|
+
|
|
4
|
+
<service>_YYYY-MM-DD.jsonl the weekly files (date = the week's Monday)
|
|
5
|
+
<service>.<stream>_YYYY-MM-DD.jsonl a longer-kept stream, with its own retention
|
|
6
|
+
<anything>.yyyyMMdd.out.log / .err.log WinSW's daily console captures (service-ops spec)
|
|
7
|
+
|
|
8
|
+
Everything else in the folder (WinSW's roll-by-size files, its wrapper log, other apps) is left alone."""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import os
|
|
13
|
+
import re
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
from datetime import date, datetime, timedelta
|
|
16
|
+
from zoneinfo import ZoneInfo
|
|
17
|
+
|
|
18
|
+
from ._time import local_date
|
|
19
|
+
|
|
20
|
+
_WINSW = re.compile(r"^.+\.(\d{4})(\d{2})(\d{2})\.(?:out|err)\.log$")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class SweepResult:
|
|
25
|
+
deleted: list[str] = field(default_factory=list)
|
|
26
|
+
failed: list[str] = field(default_factory=list)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _weekly(prefix: str) -> re.Pattern[str]:
|
|
30
|
+
return re.compile(rf"^{re.escape(prefix)}_(\d{{4}})-(\d{{2}})-(\d{{2}})\.jsonl$")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def sweep_expired(
|
|
34
|
+
directory: str,
|
|
35
|
+
service: str,
|
|
36
|
+
retention_days: int,
|
|
37
|
+
now: datetime,
|
|
38
|
+
tz: ZoneInfo,
|
|
39
|
+
streams: dict[str, int] | None = None,
|
|
40
|
+
) -> SweepResult:
|
|
41
|
+
"""`streams` maps a stream name to its own retention in days."""
|
|
42
|
+
result = SweepResult()
|
|
43
|
+
try:
|
|
44
|
+
names = os.listdir(directory)
|
|
45
|
+
except OSError:
|
|
46
|
+
return result # no folder yet: nothing to delete
|
|
47
|
+
today = local_date(now, tz)
|
|
48
|
+
rules: list[tuple[re.Pattern[str], int, int]] = [(_weekly(service), retention_days, 6)]
|
|
49
|
+
for stream, days in (streams or {}).items():
|
|
50
|
+
rules.append((_weekly(f"{service}.{stream}"), days, 6))
|
|
51
|
+
|
|
52
|
+
for name in names:
|
|
53
|
+
last_day: date | None = None
|
|
54
|
+
keep_days = retention_days
|
|
55
|
+
for pattern, days, span in rules:
|
|
56
|
+
m = pattern.match(name)
|
|
57
|
+
if m:
|
|
58
|
+
try:
|
|
59
|
+
# A weekly file holds lines up to its Sunday; keep it until all are past retention.
|
|
60
|
+
last_day = date(int(m[1]), int(m[2]), int(m[3])) + timedelta(days=span)
|
|
61
|
+
except ValueError:
|
|
62
|
+
last_day = None
|
|
63
|
+
keep_days = days
|
|
64
|
+
break
|
|
65
|
+
else:
|
|
66
|
+
m = _WINSW.match(name)
|
|
67
|
+
if m:
|
|
68
|
+
try:
|
|
69
|
+
last_day = date(int(m[1]), int(m[2]), int(m[3]))
|
|
70
|
+
except ValueError:
|
|
71
|
+
last_day = None
|
|
72
|
+
if last_day is None:
|
|
73
|
+
continue
|
|
74
|
+
if (today - last_day).days > keep_days:
|
|
75
|
+
try:
|
|
76
|
+
os.remove(os.path.join(directory, name))
|
|
77
|
+
result.deleted.append(name)
|
|
78
|
+
except OSError:
|
|
79
|
+
result.failed.append(name)
|
|
80
|
+
return result
|