logmask 0.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
logmask/__init__.py ADDED
@@ -0,0 +1,18 @@
1
+ from .config import masked_config
2
+ from .configure import setup_logging
3
+ from .excepthook import install_excepthook
4
+ from .formatter import JsonFormatter
5
+ from .handler import SafeStreamHandler
6
+ from .secrets import MASKED_VALUE, SecretsMasker
7
+ from .structlog_processor import mask_secrets_processor
8
+
9
+ __all__ = (
10
+ 'MASKED_VALUE',
11
+ 'JsonFormatter',
12
+ 'SafeStreamHandler',
13
+ 'SecretsMasker',
14
+ 'install_excepthook',
15
+ 'mask_secrets_processor',
16
+ 'masked_config',
17
+ 'setup_logging',
18
+ )
logmask/config.py ADDED
@@ -0,0 +1,29 @@
1
+ from collections.abc import Mapping
2
+ from typing import Any
3
+
4
+ from .secrets import MASKED_VALUE, check_private_value
5
+
6
+
7
+ def masked_config(public: Mapping[str, Any], private: Mapping[str, Any]) -> dict[str, Any]:
8
+ """The config for the startup log: public values as they are, private values replaced by the placeholder.
9
+
10
+ A private list keeps its length, and an unset private value (None or an empty string) becomes an empty string, so
11
+ the log shows which variables are set without reading their values. Any other value, `0` and `False` included, is
12
+ set and shows the placeholder. A variable in both mappings is an error, and so is a private value of a type the
13
+ masker cannot search for secrets, the same error `setup_logging` raises.
14
+ """
15
+ overlap = sorted(set(public) & set(private))
16
+ if overlap:
17
+ raise ValueError(f'Variables listed as both public and private: {overlap}')
18
+ for value in private.values():
19
+ check_private_value(value)
20
+
21
+ return {**public, **{key: _mask(value) for key, value in private.items()}}
22
+
23
+
24
+ def _mask(value: Any) -> Any:
25
+ if value is None or (isinstance(value, str | bytes) and not value):
26
+ return ''
27
+ if isinstance(value, list | tuple):
28
+ return [_mask(item) for item in value]
29
+ return MASKED_VALUE
logmask/configure.py ADDED
@@ -0,0 +1,51 @@
1
+ import logging
2
+ import threading
3
+ from collections.abc import Mapping
4
+ from typing import IO
5
+
6
+ from .excepthook import install_excepthook
7
+ from .formatter import JsonFormatter
8
+ from .handler import SafeStreamHandler
9
+ from .secrets import SecretsMasker
10
+
11
+ # One masker per process: a second call adds its secrets to the same masker, so the secrets of the first call stay
12
+ # masked after a library or a test configures logging again.
13
+ _masker: SecretsMasker | None = None
14
+ _lock = threading.Lock()
15
+
16
+
17
+ def setup_logging(secrets: Mapping[str, object], *, level: int | str = logging.INFO, stream: IO[str] | None = None) -> SecretsMasker:
18
+ """Configure the root logger: JSON lines to the stream (stderr by default), every line masked.
19
+
20
+ `secrets` is the mapping of private variables; a service that reads its secrets later passes `{}` and calls
21
+ `add()` on the returned masker. A private value of a type the masker cannot search raises `TypeError`. Existing
22
+ handlers of the root logger are replaced, the handler of last resort (for a logger that does not propagate and
23
+ has no handler) is replaced by a masked one, warnings are routed through the logger, and the exception hooks of
24
+ the main thread, of the other threads and of the unraisable exceptions are installed, so every traceback is
25
+ masked too. The same masker serves a structlog processor or masks text by hand. `level` is a logging level or its
26
+ name in any case.
27
+ """
28
+ global _masker
29
+ with _lock:
30
+ if _masker is None:
31
+ _masker = SecretsMasker(secrets)
32
+ else:
33
+ _masker.add(secrets)
34
+ masker = _masker
35
+ if isinstance(level, str):
36
+ level = level.upper()
37
+
38
+ handler = SafeStreamHandler(stream, masker=masker)
39
+ handler.setFormatter(JsonFormatter(masker))
40
+ logging.basicConfig(level=level, handlers=[handler], force=True)
41
+ # A logger with propagate=False and no handler of its own goes to logging.lastResort, which writes the raw
42
+ # message to stderr. The replacement writes a masked JSON line to the same stream, at the same level (WARNING).
43
+ last_resort = SafeStreamHandler(stream, masker=masker)
44
+ last_resort.setFormatter(JsonFormatter(masker))
45
+ last_resort.setLevel(logging.WARNING)
46
+ logging.lastResort = last_resort
47
+ # warnings.warn() prints to stderr by default; a warning text may carry a URL.
48
+ logging.captureWarnings(True)
49
+
50
+ install_excepthook(masker=masker)
51
+ return masker
logmask/excepthook.py ADDED
@@ -0,0 +1,102 @@
1
+ # sys.UnraisableHookArgs exists only in the type stubs: the annotations must not be evaluated at runtime.
2
+ from __future__ import annotations
3
+
4
+ import contextlib
5
+ import logging
6
+ import sys
7
+ import threading
8
+ import traceback
9
+ from types import TracebackType
10
+
11
+ from .formatter import JsonFormatter
12
+ from .handler import SafeStreamHandler
13
+ from .secrets import LIBRARY_TAG, SecretsMasker
14
+
15
+ LOGGER_NAME = 'unhandled'
16
+
17
+
18
+ def _format(exc_type: type[BaseException], exc_value: BaseException | None, exc_traceback: TracebackType | None) -> str:
19
+ if exc_value is None:
20
+ return f'{exc_type.__name__}\n' + ''.join(traceback.format_tb(exc_traceback))
21
+ return ''.join(traceback.format_exception(exc_type, exc_value, exc_traceback))
22
+
23
+
24
+ def _log_exception(
25
+ exc_type: type[BaseException],
26
+ exc_value: BaseException | None,
27
+ exc_traceback: TracebackType | None,
28
+ *,
29
+ masker: SecretsMasker,
30
+ message: str,
31
+ thread_name: str | None = None,
32
+ context: str | None = None,
33
+ ) -> None:
34
+ """Send the traceback through the logger, so that it is formatted and masked too.
35
+
36
+ A stop by Ctrl-C is not an error: for KeyboardInterrupt the traceback goes to stderr as plain text, like the
37
+ default hook prints it, but masked. The chained exceptions in it (`__context__`) may carry a URL or a key.
38
+ The traceback text is masked before it enters the record: a handler that is not the masking one (added by a
39
+ library after the setup) would otherwise print it raw. When the logger has no handler, the line is written to
40
+ stderr by this module, masked, instead of the raw record that `logging.lastResort` would print. When anything in
41
+ here fails, one masked line goes to stderr: the interpreter would otherwise print the hook's error together with
42
+ the original traceback, unmasked.
43
+ """
44
+ try:
45
+ error = masker.mask(_format(exc_type, exc_value, exc_traceback))
46
+ if issubclass(exc_type, KeyboardInterrupt):
47
+ sys.stderr.write(error)
48
+ return
49
+
50
+ record_message: dict[str, object] = {'msg': message, 'error': error}
51
+ if thread_name is not None:
52
+ record_message['thread'] = masker.mask(thread_name)
53
+ if context is not None:
54
+ record_message['context'] = masker.mask(context)
55
+
56
+ logger = logging.getLogger(LOGGER_NAME)
57
+ if logger.hasHandlers():
58
+ logger.critical(record_message)
59
+ return
60
+ record = logger.makeRecord(logger.name, logging.CRITICAL, '(unknown file)', 0, record_message, (), None)
61
+ handler = SafeStreamHandler(sys.stderr, masker=masker)
62
+ handler.setFormatter(JsonFormatter(masker))
63
+ handler.handle(record)
64
+ except BaseException as failure: # noqa: BLE001 - the hook must not raise, see the docstring
65
+ with contextlib.suppress(Exception):
66
+ sys.stderr.write(masker.mask(f'--- {LIBRARY_TAG} {exc_type.__name__} could not be logged: {type(failure).__name__} ---\n'))
67
+
68
+
69
+ def install_excepthook(*, masker: SecretsMasker) -> None:
70
+ """Route the unhandled exceptions of the main thread, of the other threads, and the unraisable ones through the logger.
71
+
72
+ `masker` covers the output that cannot go through the logger: a KeyboardInterrupt traceback, any exception when
73
+ no handler is configured, and a failure of the hook itself. The logger's own handler masks the rest.
74
+ """
75
+
76
+ def hook(exc_type: type[BaseException], exc_value: BaseException, exc_traceback: TracebackType | None) -> None:
77
+ _log_exception(exc_type, exc_value, exc_traceback, masker=masker, message='Unhandled exception. Shut down.')
78
+
79
+ def thread_hook(args: threading.ExceptHookArgs) -> None:
80
+ if issubclass(args.exc_type, SystemExit):
81
+ # The default thread hook ignores sys.exit() in a thread; so do we.
82
+ return
83
+ thread_name = args.thread.name if args.thread is not None else None
84
+ _log_exception(
85
+ args.exc_type,
86
+ args.exc_value,
87
+ args.exc_traceback,
88
+ masker=masker,
89
+ message='Unhandled exception in a thread.',
90
+ thread_name=thread_name,
91
+ )
92
+
93
+ def unraisable_hook(args: sys.UnraisableHookArgs) -> None:
94
+ # An exception in __del__, in a weakref callback or in a GC hook: the default hook prints it to stderr.
95
+ # err_msg says where, when the interpreter sets it (Python 3.14 and later); otherwise the type of the object
96
+ # stands in. The object itself is not logged, its repr is unknown text.
97
+ context = args.err_msg or f'Exception ignored in: {type(args.object).__name__}'
98
+ _log_exception(args.exc_type, args.exc_value, args.exc_traceback, masker=masker, message='Unraisable exception.', context=context)
99
+
100
+ sys.excepthook = hook
101
+ threading.excepthook = thread_hook
102
+ sys.unraisablehook = unraisable_hook
logmask/formatter.py ADDED
@@ -0,0 +1,107 @@
1
+ import dataclasses
2
+ import json
3
+ import logging
4
+ from collections.abc import Callable, Iterator, Mapping
5
+ from typing import Any
6
+
7
+ from .secrets import LIBRARY_TAG, SecretsMasker
8
+
9
+ UNPRINTABLE = '<unprintable>'
10
+
11
+
12
+ def json_default(value: object) -> Any:
13
+ """Values json.dumps does not know: bytes as 0x-hex, dataclasses as dicts, sets and iterators as lists, the rest as text."""
14
+ if isinstance(value, bytes | bytearray):
15
+ return '0x' + bytes(value).hex()
16
+ if dataclasses.is_dataclass(value) and not isinstance(value, type):
17
+ return dataclasses.asdict(value)
18
+ if isinstance(value, Mapping):
19
+ return dict(value)
20
+ if isinstance(value, set | frozenset | Iterator):
21
+ return list(value)
22
+ return str(value)
23
+
24
+
25
+ def _safe(convert: Callable[[Any], str], value: Any) -> str:
26
+ try:
27
+ return convert(value)
28
+ except Exception: # noqa: BLE001 - a __str__ or __repr__ that raises must not break the fallback line
29
+ return UNPRINTABLE
30
+
31
+
32
+ class JsonFormatter(logging.Formatter):
33
+ """One JSON object per line with a fixed set of record fields; every line goes through the masker.
34
+
35
+ `format()` never raises: a record that cannot be formatted becomes a line with an `error` field. Otherwise the
36
+ handler would report the error with the raw message and arguments, without masking, or the error would reach the
37
+ caller of the logger.
38
+ """
39
+
40
+ def __init__(self, masker: SecretsMasker):
41
+ super().__init__()
42
+ self._masker = masker
43
+
44
+ def format(self, record: logging.LogRecord) -> str:
45
+ try:
46
+ text = json.dumps(self._line(record), default=json_default)
47
+ except Exception as error: # noqa: BLE001 - any formatting error must become a masked line, see the class docstring
48
+ text = json.dumps(self._fallback_line(record, error))
49
+ # Secrets are hidden in every log line, including messages from third-party libraries.
50
+ masked = self._masker.mask(text)
51
+ try:
52
+ json.loads(masked)
53
+ return masked
54
+ except ValueError:
55
+ pass
56
+ # A secret made of JSON syntax (a quote, a brace, `true`, a number) was replaced outside the strings. Mask
57
+ # the strings one by one instead; the JSON-escaped forms are then not needed, and the syntax stays.
58
+ try:
59
+ return json.dumps(self._mask_leaves(json.loads(text)))
60
+ except Exception as error: # noqa: BLE001 - a message nested too deep for the recursion limit, see the class docstring
61
+ # The fallback line is flat: its values are the record fields and short texts.
62
+ return json.dumps(self._mask_leaves(self._fallback_line(record, error)))
63
+
64
+ def _mask_leaves(self, value: Any) -> Any:
65
+ if isinstance(value, str):
66
+ return self._masker.mask(value)
67
+ if isinstance(value, dict):
68
+ return {self._masker.mask(key): self._mask_leaves(item) for key, item in value.items()}
69
+ if isinstance(value, list):
70
+ return [self._mask_leaves(item) for item in value]
71
+ if isinstance(value, int | float) and not isinstance(value, bool):
72
+ # A number equal to a secret (a PIN, an id) is what broke the line: it is replaced by the masked text.
73
+ text = str(value)
74
+ masked = self._masker.mask(text)
75
+ return value if masked == text else masked
76
+ return value
77
+
78
+ def _base(self, record: logging.LogRecord) -> dict[str, Any]:
79
+ return {
80
+ 'name': record.name,
81
+ 'levelname': record.levelname,
82
+ 'funcName': record.funcName,
83
+ 'lineno': record.lineno,
84
+ 'module': record.module,
85
+ 'pathname': record.pathname,
86
+ 'timestamp': int(record.created),
87
+ }
88
+
89
+ def _line(self, record: logging.LogRecord) -> dict[str, Any]:
90
+ message = record.msg if isinstance(record.msg, dict) else {'msg': record.getMessage()}
91
+ line = self._base(record)
92
+ for key, value in message.items():
93
+ # The record fields win: a message key `name` or `timestamp` would break every query on them.
94
+ line[f'msg_{key}' if key in line else key] = value
95
+ if record.exc_info:
96
+ line['exc_info'] = self.formatException(record.exc_info)
97
+ return line
98
+
99
+ def _fallback_line(self, record: logging.LogRecord, error: Exception) -> dict[str, Any]:
100
+ line = {
101
+ **self._base(record),
102
+ 'msg': _safe(str, record.msg),
103
+ 'error': f'{LIBRARY_TAG} Log record could not be formatted: {_safe(repr, error)}',
104
+ }
105
+ if record.args:
106
+ line['args'] = _safe(repr, record.args)
107
+ return line
logmask/handler.py ADDED
@@ -0,0 +1,28 @@
1
+ import contextlib
2
+ import logging
3
+ import sys
4
+ from typing import IO
5
+
6
+ from .secrets import LIBRARY_TAG, SecretsMasker
7
+
8
+
9
+ class SafeStreamHandler(logging.StreamHandler): # type: ignore[type-arg]
10
+ """A StreamHandler whose error report does not show the record.
11
+
12
+ When a handler cannot write a record (a closed stream, a broken pipe), the standard `handleError` prints the raw
13
+ message and the arguments to stderr, without masking. This handler reports the error and where the record came
14
+ from, masked, and nothing else from the record.
15
+ """
16
+
17
+ def __init__(self, stream: IO[str] | None = None, *, masker: SecretsMasker):
18
+ super().__init__(stream)
19
+ self._masker = masker
20
+
21
+ def handleError(self, record: logging.LogRecord) -> None:
22
+ if not logging.raiseExceptions:
23
+ return
24
+ error = sys.exc_info()[1]
25
+ text = f'--- {LIBRARY_TAG} Logging error: {error!r} (record from {record.name}, {record.pathname}:{record.lineno}) ---\n'
26
+ # stderr may be the stream that failed; there is nowhere else to report.
27
+ with contextlib.suppress(Exception):
28
+ sys.stderr.write(self._masker.mask(text))
logmask/py.typed ADDED
File without changes
logmask/secrets.py ADDED
@@ -0,0 +1,302 @@
1
+ import base64
2
+ import dataclasses
3
+ import json
4
+ import numbers
5
+ import os
6
+ import re
7
+ import threading
8
+ from collections.abc import Iterable, Iterator, Mapping
9
+ from urllib.parse import quote, quote_plus, unquote, unquote_plus, urlsplit
10
+
11
+ MASKED_VALUE = '***'
12
+ # Every line the library writes on its own behalf, instead of the record, starts with this tag.
13
+ LIBRARY_TAG = '[logmask]'
14
+ # Every private value is masked whole, whatever its length. A standalone fragment of a private URL (the host, one
15
+ # path segment, one query value) is masked on its own only when it can be a key: 16 characters or more, or 8 or more
16
+ # with a digit. The whole URL, the whole path, the whole query and the user:password are masked without any
17
+ # condition. Masking a word like `eth`, a segment like `v3` or a host like `el` on its own would erase it from every
18
+ # log line.
19
+ _KEY_FRAGMENT_LENGTH = 16
20
+ _KEY_FRAGMENT_LENGTH_WITH_DIGIT = 8
21
+ # The encodings of a value (percent, base64, hex of its bytes) are derived for values of 4 characters or more: the
22
+ # encodings of a shorter value are short random-looking strings that hit ordinary words. The value itself is always
23
+ # masked.
24
+ _ENCODED_FORMS_MIN_LENGTH = 4
25
+ # How many layers of repr() or json.dumps() around a secret are recognised, on top of the JSON encoding of the line.
26
+ _ESCAPE_DEPTH = 2
27
+
28
+ _HEX = re.compile(r'(0x)?([0-9a-f]+)', re.ASCII | re.IGNORECASE)
29
+
30
+
31
+ def _key_fragment(text: str) -> bool:
32
+ return len(text) >= _KEY_FRAGMENT_LENGTH or (len(text) >= _KEY_FRAGMENT_LENGTH_WITH_DIGIT and any(char.isdigit() for char in text))
33
+
34
+
35
+ def _utf8(text: str) -> bytes:
36
+ # A value read from the environment may carry bytes that are not UTF-8, as lone surrogates (surrogateescape).
37
+ return text.encode('utf-8', errors='surrogateescape')
38
+
39
+
40
+ def _hex_forms(text: str) -> Iterator[str]:
41
+ # A hex secret is logged with or without the 0x prefix, and as the bytes it encodes (a private key is held as
42
+ # bytes by eth_account); the pattern ignores the case.
43
+ match = _HEX.fullmatch(text)
44
+ if match is None:
45
+ return
46
+ digits = match[2]
47
+ yield digits
48
+ yield '0x' + digits
49
+ if len(digits) % 2 == 0:
50
+ yield repr(bytes.fromhex(digits))[2:-1]
51
+
52
+
53
+ def _encoded_forms(text: str) -> Iterator[str]:
54
+ """The forms an application gives a raw secret when it builds a URL, a header or a bytes value from it."""
55
+ if len(text) < _ENCODED_FORMS_MIN_LENGTH:
56
+ return
57
+ raw = _utf8(text)
58
+ yield quote(text, safe='', errors='surrogateescape')
59
+ yield quote(text, errors='surrogateescape')
60
+ yield quote_plus(text, errors='surrogateescape')
61
+ for encoded in (base64.b64encode(raw), base64.urlsafe_b64encode(raw)):
62
+ yield encoded.decode()
63
+ yield encoded.decode().rstrip('=')
64
+ # json_default writes a bytes value as 0x-hex; a text secret inside such a value must be found in that form too.
65
+ yield raw.hex()
66
+ yield repr(raw)[2:-1]
67
+
68
+
69
+ def _with_decoded(text: str) -> Iterator[str]:
70
+ yield text
71
+ yield unquote(text)
72
+
73
+
74
+ def _url_parts(text: str) -> Iterator[str]:
75
+ try:
76
+ url = urlsplit(text)
77
+ except ValueError:
78
+ return
79
+ if not (url.scheme and url.netloc):
80
+ return
81
+
82
+ # The netloc with the credentials always, and the host alone when a provider may have put the key into it.
83
+ if '@' in url.netloc:
84
+ yield url.netloc
85
+ for host in (url.netloc.rpartition('@')[2], url.hostname or ''):
86
+ if _key_fragment(host):
87
+ yield host
88
+ for part in (url.username, url.password):
89
+ if part:
90
+ yield from _with_decoded(part)
91
+ if url.password:
92
+ userinfo = f'{url.username or ""}:{url.password}'
93
+ yield from _with_decoded(userinfo)
94
+ # The same credentials as an HTTP Basic header.
95
+ yield base64.b64encode(_utf8(unquote(userinfo))).decode()
96
+
97
+ # HTTP libraries log the path, or the path with the query, without the host.
98
+ if url.path.strip('/'):
99
+ yield from _with_decoded(url.path)
100
+ yield url.path.rstrip('/')
101
+ for segment in url.path.split('/'):
102
+ if _key_fragment(segment):
103
+ yield from _with_decoded(segment)
104
+ if url.query:
105
+ yield url.query
106
+ yield f'{url.path}?{url.query}'
107
+ for pair in url.query.split('&'):
108
+ yield pair
109
+ value = pair.partition('=')[2]
110
+ if _key_fragment(value):
111
+ yield from _with_decoded(value)
112
+
113
+
114
+ def _variants(text: str) -> Iterator[str]:
115
+ # A secret read from a file usually keeps the trailing newline, the log shows the stripped value.
116
+ for form in dict.fromkeys((text, text.strip())):
117
+ yield form
118
+ yield form.rstrip('/')
119
+ # A value stored percent-encoded (a password inside a URL) is used decoded by the application.
120
+ yield unquote(form)
121
+ yield unquote_plus(form)
122
+ yield from _hex_forms(form)
123
+ yield from _url_parts(form)
124
+ yield from _encoded_forms(form)
125
+
126
+
127
+ def _encodings(text: str) -> Iterator[str]:
128
+ """The body of a string after one layer of repr() or json.dumps()."""
129
+ # repr() picks the quote style from the whole string: a secret with a ' inside a string that also has a " is
130
+ # escaped as \'. Both bodies are needed.
131
+ yield repr(text)[1:-1]
132
+ yield repr(text + '"')[1:-2]
133
+ yield json.dumps(text)[1:-1]
134
+
135
+
136
+ def _escaped_forms(text: str) -> set[str]:
137
+ forms = {text}
138
+ layer = {text}
139
+ for _ in range(_ESCAPE_DEPTH):
140
+ layer = {encoded for form in layer for encoded in _encodings(form)}
141
+ forms |= layer
142
+ return forms
143
+
144
+
145
+ SUPPORTED_TYPES = 'str, bytes, a number, a path, a mapping, a list or another iterable, a dataclass or an object with get_secret_value()'
146
+
147
+
148
+ def _strings_inside(text: str, seen: dict[int, object]) -> Iterator[str]:
149
+ """The values held by one string: a JSON document (a list of URLs, the JSON of a service account) is searched as
150
+ the container it encodes, several URLs separated by commas, semicolons or white space are searched one by one
151
+ (otherwise only the last URL would get its parts, the others would be plain prefixes of the value), and every
152
+ line of a multi-line value (a PEM key) is a secret on its own, as a library prints it line by line."""
153
+ if text.lstrip()[:1] in ('[', '{'):
154
+ try:
155
+ document = json.loads(text)
156
+ except ValueError:
157
+ document = None
158
+ if isinstance(document, list | dict):
159
+ yield from _strings(document, seen)
160
+ return
161
+ urls = [piece for piece in re.split(r'[,;\s]+', text) if '://' in piece]
162
+ if len(urls) > 1:
163
+ yield from urls
164
+ lines = [line.strip() for line in text.splitlines()]
165
+ if len(lines) > 1:
166
+ yield from (line for line in lines if line)
167
+
168
+
169
+ def _strings(value: object, seen: dict[int, object]) -> Iterator[str]:
170
+ """Every string inside the value: strings and the values a string holds (a JSON document, a list of URLs), bytes
171
+ in their text, hex and repr forms, the items of containers and other iterables (the keys of a nested mapping are
172
+ data, like its values; the keys of the top-level mapping are the variable names and never reach here), the fields
173
+ of a dataclass, the wrapped value of a SecretStr-like object, numbers and paths as text. None and bool are not
174
+ secrets. Any other type is refused: the masker cannot promise that it found every secret inside it.
175
+
176
+ `seen` keeps a reference to every container visited, so that a cycle stops and a freed temporary cannot hand its
177
+ id to a later container that would then be skipped.
178
+ """
179
+ if value is None or isinstance(value, bool):
180
+ return
181
+ if isinstance(value, str):
182
+ yield value
183
+ yield from _strings_inside(value, seen)
184
+ elif isinstance(value, bytes | bytearray | memoryview):
185
+ raw = bytes(value)
186
+ yield raw.decode('utf-8', errors='surrogateescape')
187
+ yield raw.hex()
188
+ yield repr(raw)[2:-1]
189
+ elif isinstance(value, numbers.Number | os.PathLike):
190
+ # Whatever text the value shows in a log is a secret too.
191
+ yield str(value)
192
+ elif id(value) in seen:
193
+ return
194
+ elif isinstance(value, Mapping):
195
+ seen[id(value)] = value
196
+ for key, item in value.items():
197
+ yield from _strings(key, seen)
198
+ yield from _strings(item, seen)
199
+ elif isinstance(value, Iterable):
200
+ seen[id(value)] = value
201
+ for item in value:
202
+ yield from _strings(item, seen)
203
+ elif callable(getter := getattr(value, 'get_secret_value', None)):
204
+ # pydantic SecretStr and the like: str() shows a placeholder, the secret is behind the method.
205
+ seen[id(value)] = value
206
+ yield from _strings(getter(), seen)
207
+ elif dataclasses.is_dataclass(value):
208
+ seen[id(value)] = value
209
+ for field in dataclasses.fields(value):
210
+ yield from _strings(getattr(value, field.name), seen)
211
+ else:
212
+ raise TypeError(f'A private value of type {type(value).__name__} is not supported, pass {SUPPORTED_TYPES}')
213
+
214
+
215
+ def check_private_value(value: object) -> None:
216
+ """Raise `TypeError` for a value the masker cannot search for secrets, by the rule of `SecretsMasker`."""
217
+ for _ in _strings(value, {}):
218
+ pass
219
+
220
+
221
+ def _secret_values(secrets: Mapping[str, object]) -> frozenset[str]:
222
+ """Every string to hide in the formatted log line, derived from the secret values."""
223
+ forms: set[str] = set()
224
+ for value in secrets.values():
225
+ for item in _strings(value, {}):
226
+ for variant in _variants(item):
227
+ if variant:
228
+ forms |= _escaped_forms(variant)
229
+ # Every form once more JSON-escaped: the replacement runs on the formatted JSON line.
230
+ return frozenset(forms | {json.dumps(form)[1:-1] for form in forms})
231
+
232
+
233
+ def _compile(forms: Iterable[str], flags: int) -> re.Pattern[str]:
234
+ # Longest first, so that a value which contains another one is replaced as a whole. The alternation tries the
235
+ # values in order, and the lookahead makes the search report a match at every position, also inside an earlier
236
+ # match: overlapping secrets are merged into one span, a plain substitution would leave the tail of the second
237
+ # one visible.
238
+ ordered = sorted(set(forms), key=lambda form: (-len(form), form))
239
+ pattern = re.compile('(?=(' + '|'.join(map(re.escape, ordered)) + '))', flags)
240
+ # re keeps the last few hundred compiled patterns in a process-wide cache, keyed by their source. Every add()
241
+ # compiles a bigger pattern, and the earlier ones, with the text of the secrets, would stay in that cache for
242
+ # the life of the process: hundreds of MiB after a few hundred token rotations, and secrets that outlive the
243
+ # masker. The cache is dropped; other modules only recompile their patterns once.
244
+ re.purge()
245
+ return pattern
246
+
247
+
248
+ class SecretsMasker:
249
+ """Replaces every form of the secret values with the masked value in any text, for example in a formatted log line.
250
+
251
+ The case of the text does not matter: a key is logged upper-case as often as lower-case. New secrets can be added
252
+ later with `add()`, for a service that configures logging before it reads its config. A value of a type the masker
253
+ cannot search (`SUPPORTED_TYPES` lists the supported ones) raises `TypeError`, in the constructor and in `add()`.
254
+ """
255
+
256
+ def __init__(self, secrets: Mapping[str, object]):
257
+ self._lock = threading.Lock()
258
+ self._forms: frozenset[str] = frozenset()
259
+ # One pattern for ASCII text, matched on the lower-cased text (same offsets, half the cost), and one
260
+ # case-insensitive pattern for text whose lower case may change the offsets.
261
+ self._pattern_ascii: re.Pattern[str] | None = None
262
+ self._pattern: re.Pattern[str] | None = None
263
+ self.add(secrets)
264
+
265
+ def add(self, secrets: Mapping[str, object]) -> None:
266
+ """Mask the values of these secrets too, in every line from now on."""
267
+ new_forms = _secret_values(secrets)
268
+ with self._lock:
269
+ forms = self._forms | new_forms
270
+ if forms == self._forms:
271
+ return
272
+ pattern_ascii = _compile((form.lower() for form in forms), 0)
273
+ pattern = _compile(forms, re.IGNORECASE)
274
+ self._forms, self._pattern_ascii, self._pattern = forms, pattern_ascii, pattern
275
+
276
+ def __repr__(self) -> str:
277
+ return f'{type(self).__name__}({len(self._forms)} forms)'
278
+
279
+ def mask(self, text: str) -> str:
280
+ pattern_ascii, pattern = self._pattern_ascii, self._pattern
281
+ if pattern is None or pattern_ascii is None:
282
+ return text
283
+ matches = pattern_ascii.finditer(text.lower()) if text.isascii() else pattern.finditer(text)
284
+
285
+ spans: list[list[int]] = []
286
+ for match in matches:
287
+ start, end = match.start(), match.start() + len(match[1])
288
+ if spans and start <= spans[-1][1]:
289
+ spans[-1][1] = max(spans[-1][1], end)
290
+ else:
291
+ spans.append([start, end])
292
+ if not spans:
293
+ return text
294
+
295
+ parts: list[str] = []
296
+ last = 0
297
+ for start, end in spans:
298
+ parts.append(text[last:start])
299
+ parts.append(MASKED_VALUE)
300
+ last = end
301
+ parts.append(text[last:])
302
+ return ''.join(parts)
@@ -0,0 +1,27 @@
1
+ from collections.abc import Callable
2
+ from typing import Any
3
+
4
+ from .secrets import SecretsMasker
5
+
6
+ Processor = Callable[[Any, str, Any], Any]
7
+
8
+
9
+ def mask_secrets_processor(masker: SecretsMasker) -> Processor:
10
+ """A structlog processor that masks the rendered line.
11
+
12
+ Put it last, after the renderer: it then masks the whole line, keys and JSON-escaped forms included. It accepts
13
+ the output of a renderer, a `str` or `bytes`, and raises TypeError for anything else (an event dict): placed
14
+ before the renderer it could not mask the objects inside the event, and a silent partial masking is worse than a
15
+ failure at the first log call. The module does not import structlog: the processor is a plain function with the
16
+ `(logger, method_name, event_dict)` contract, the service brings its own structlog.
17
+ """
18
+
19
+ def processor(logger: Any, method_name: str, event: Any) -> Any:
20
+ if isinstance(event, str):
21
+ return masker.mask(event)
22
+ if isinstance(event, bytes):
23
+ text = event.decode('utf-8', errors='surrogateescape')
24
+ return masker.mask(text).encode('utf-8', errors='surrogateescape')
25
+ raise TypeError(f'mask_secrets_processor must be the last processor, after the renderer; got {type(event).__name__}')
26
+
27
+ return processor
@@ -0,0 +1,379 @@
1
+ Metadata-Version: 2.4
2
+ Name: logmask
3
+ Version: 0.1.3
4
+ Summary: JSON logging with masked secrets for Python services
5
+ License-Expression: MIT
6
+ License-File: LICENSE
7
+ Author: Lido
8
+ Requires-Python: >=3.12,<4
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Programming Language :: Python :: 3.12
11
+ Classifier: Programming Language :: Python :: 3.13
12
+ Classifier: Programming Language :: Python :: 3.14
13
+ Classifier: Programming Language :: Python :: 3.15
14
+ Project-URL: Repository, https://github.com/lidofinance/logmask
15
+ Description-Content-Type: text/markdown
16
+
17
+ # logmask
18
+
19
+ JSON logging for Python services, with secrets masked. Every log line is one JSON object. Every line goes through a masker that replaces every form of your secret values with `***`. This includes the messages of third-party libraries, warnings, and the traceback of an unhandled exception.
20
+
21
+ No runtime dependencies. Python 3.12 or newer.
22
+
23
+ ## Install
24
+
25
+ ```bash
26
+ poetry add git+https://github.com/lidofinance/logmask.git@v0.1.3
27
+ ```
28
+
29
+ Pin a tag, or a commit with `rev`, and commit `poetry.lock`: it records the exact commit. Poetry clones the repository itself; a build stage that installs with `pip` needs `git`.
30
+
31
+ ## Quick start
32
+
33
+ ```py
34
+ import logging
35
+
36
+ from logmask import masked_config, setup_logging
37
+
38
+ PUBLIC = {'RPC_TIMEOUT': 30, 'DRY_RUN': True}
39
+ PRIVATE = {'RPC_URL': 'https://rpc.example.com/v3/a1b2c3d4e5f6', 'API_TOKEN': 'tok_9f8e7d6c5b4a'}
40
+
41
+ setup_logging(PRIVATE)
42
+
43
+ logger = logging.getLogger(__name__)
44
+ logger.info({'msg': 'Config', 'value': masked_config(PUBLIC, PRIVATE)})
45
+ logger.info('Connecting to %s', PRIVATE['RPC_URL'])
46
+ ```
47
+
48
+ Output (one line per record, stderr):
49
+
50
+ ```json
51
+ {"name": "app", "levelname": "INFO", "funcName": "main", "lineno": 12, "module": "main", "pathname": "/app/main.py", "timestamp": 1761600000, "msg": "Config", "value": {"RPC_TIMEOUT": 30, "DRY_RUN": true, "RPC_URL": "***", "API_TOKEN": "***"}}
52
+ {"name": "app", "levelname": "INFO", "funcName": "main", "lineno": 13, "module": "main", "pathname": "/app/main.py", "timestamp": 1761600000, "msg": "Connecting to ***"}
53
+ ```
54
+
55
+ `setup_logging(PRIVATE)` does four things:
56
+
57
+ - configures the root logger: JSON lines to stderr, level INFO, old handlers removed
58
+ - masks every line with every form of the values of `PRIVATE`
59
+ - routes `warnings.warn()` through the logger, so a warning text is masked too
60
+ - installs `sys.excepthook`, `threading.excepthook` and `sys.unraisablehook`, so an unhandled exception becomes one CRITICAL line with a masked traceback
61
+
62
+ `masked_config(PUBLIC, PRIVATE)` builds a dict for the startup log. Public values stay as they are. Private values become `***`. A private list keeps its length. An unset private value (`None`, `''` or `b''`) becomes `''`, so you can see which variables are set; any other value, `0` and `False` included, is set and shows `***`. A key present in both dicts raises `ValueError`, and a private value of a type the masker cannot search raises `TypeError` (see "What is masked").
63
+
64
+ ```py
65
+ masked_config({'PORT': 9000}, {'TOKEN': 'abc', 'RPC_URLS': ['a', 'b'], 'OPTIONAL_KEY': None})
66
+ # {'PORT': 9000, 'TOKEN': '***', 'RPC_URLS': ['***', '***'], 'OPTIONAL_KEY': ''}
67
+ ```
68
+
69
+ Every value of `PRIVATE` is treated as a secret, whatever it looks like (`None` and booleans are not values). Put a variable there only when its value must never appear in a log: a value such as `true` or `9000` in `PRIVATE` would be masked inside every line, field names included. A realistic set of secrets (about 20 variables, about 100 forms) costs about 50 microseconds per line; hundreds of secrets make every line noticeably slower.
70
+
71
+ ## How to
72
+
73
+ ### Log a message
74
+
75
+ A string message goes to the `msg` field. A dict message is merged into the line, so its keys become fields:
76
+
77
+ ```py
78
+ logger.info('Report sent for slot %d', 1234567)
79
+ # {..., "msg": "Report sent for slot 1234567"}
80
+
81
+ logger.info({'msg': 'Report sent', 'slot': 1234567, 'root': b'\x0a\x0b'})
82
+ # {..., "msg": "Report sent", "slot": 1234567, "root": "0x0a0b"}
83
+ ```
84
+
85
+ ### Log an exception
86
+
87
+ ```py
88
+ try:
89
+ client.get(url)
90
+ except ConnectionError:
91
+ logger.exception('Request failed')
92
+ # {..., "levelname": "ERROR", "msg": "Request failed", "exc_info": "Traceback (most recent call last):\n ...\nConnectionError: Max retries exceeded with url: ***"}
93
+ ```
94
+
95
+ ### Set the level and the stream
96
+
97
+ ```py
98
+ setup_logging(PRIVATE, level=os.getenv('LOG_LEVEL', 'INFO'), stream=sys.stdout)
99
+ ```
100
+
101
+ `level` takes a name in any case or an int. `stream` is stderr by default. Three outputs always go to stderr: the traceback of a KeyboardInterrupt, the report of a handler that cannot write, and the line written when the exception hook itself fails.
102
+
103
+ ### Add secrets later
104
+
105
+ `setup_logging` returns the masker. Pass `{}` and call `add()` when a secret becomes known after logging is configured, for example a token fetched from a vault or a config module that logs at import time:
106
+
107
+ ```py
108
+ masker = setup_logging({})
109
+
110
+ from app import config # logs at import time, before the config is known
111
+
112
+ masker.add(config.PRIVATE)
113
+ ```
114
+
115
+ Every line from that point on is masked with the new values too. The lines written before `add()` are not. A second `setup_logging` call in the same process adds its secrets to the same masker, so the secrets of the first call stay masked.
116
+
117
+ ### Mask text by hand
118
+
119
+ ```py
120
+ masker = setup_logging(PRIVATE)
121
+ masker.mask('connect https://rpc.example.com/v3/a1b2c3d4e5f6')
122
+ # 'connect ***'
123
+ ```
124
+
125
+ ### Use structlog
126
+
127
+ Call `setup_logging` as well: structlog does not route the standard-library loggers of `web3`, `urllib3` or `requests` anywhere, and without a configured root logger their warnings go to stderr unmasked.
128
+
129
+ ```py
130
+ import structlog
131
+
132
+ from logmask import mask_secrets_processor, setup_logging
133
+
134
+ masker = setup_logging(PRIVATE)
135
+ structlog.configure(
136
+ processors=[
137
+ structlog.processors.add_log_level,
138
+ structlog.processors.TimeStamper(fmt='iso'),
139
+ structlog.processors.JSONRenderer(),
140
+ mask_secrets_processor(masker), # last: masks the rendered line
141
+ ],
142
+ )
143
+ ```
144
+
145
+ The processor must be the last one, after the renderer: it masks the rendered line, keys and JSON-escaped forms included. Placed earlier it receives the event dict and raises `TypeError` at the first event that reaches the processors, which with a filtering logger is the first event at or above the configured level, not the first call. Do not render tracebacks with `structlog.processors.dict_tracebacks` in its default form: it prints the local variables of every frame cut to 80 characters, and a cut secret is a part that no form matches. Use `format_exc_info`, or `ExceptionRenderer(ExceptionDictTransformer(show_locals=False))`. The package does not depend on structlog: your service installs its own version.
146
+
147
+ ### Use the pieces one by one
148
+
149
+ `setup_logging` wires these classes; they can be used directly, and each of them requires the masker:
150
+
151
+ - `SecretsMasker(secrets)` - the masker; `mask(text)` and `add(secrets)`
152
+ - `JsonFormatter(masker)` - one JSON object per record, masked
153
+ - `SafeStreamHandler(stream, masker=masker)` - a `StreamHandler` that does not print the record when the write fails
154
+ - `install_excepthook(masker=masker)` - the three exception hooks; the masker covers the output that cannot go through the logger
155
+
156
+ ## What is masked
157
+
158
+ The examples use this masker:
159
+
160
+ ```py
161
+ masker = SecretsMasker(
162
+ {
163
+ 'RPC_URL': 'https://rpc.example.com/v3/a1b2c3d4e5f6/',
164
+ 'CL_URL': 'https://user:s3cretPassw0rd@beacon.example.com/',
165
+ 'API_TOKEN': 'tok_9f8e7d6c5b4a\n', # read from a file, with the trailing newline
166
+ 'RPC_LIST': ['https://one.example.com/key/abcdef123456', 'https://two.example.com/key/ghijkl789012'],
167
+ 'KEYS': {'nested-secret-key': 'nested-secret-value'},
168
+ 'PRIVATE_KEY': '0xABCDEF0123456789',
169
+ 'RAW_KEY': b'\xde\xad\xbe\xef\x01\x02\x03\x04',
170
+ }
171
+ )
172
+ ```
173
+
174
+ Each value, each item of a list, tuple, set or any other iterable, the keys and the values of a nested dict (only the top-level keys are the variable names, they stay), the fields of a dataclass, the value behind a `SecretStr`-like object (`get_secret_value()`), numbers and paths by their text:
175
+
176
+ ```py
177
+ masker.mask('connect https://rpc.example.com/v3/a1b2c3d4e5f6/')
178
+ # 'connect ***'
179
+ masker.mask('rpc https://two.example.com/key/ghijkl789012 failed')
180
+ # 'rpc *** failed'
181
+ masker.mask('KEYS nested-secret-key nested-secret-value')
182
+ # 'KEYS *** ***'
183
+ ```
184
+
185
+ A value of any other type (a plain object, a function, a module, a class) raises `TypeError` in `setup_logging`, `SecretsMasker`, `add()` and `masked_config`: the masker cannot promise to find every secret inside it, so it refuses the value instead of masking a part. Pass the data of such an object as a dict.
186
+
187
+ The values held by one string: a JSON document is searched as the container it encodes (a list of URLs, the JSON of a service account with its `private_key`), several URLs separated by commas, semicolons or white space are searched one by one, and every line of a multi-line value (a PEM key) is a secret on its own:
188
+
189
+ ```py
190
+ masker = SecretsMasker({'RPC_LIST': 'https://one.example.com/key/abcdef123456,https://two.example.com/key/ghijkl789012'})
191
+ masker.mask('rpc https://one.example.com/key/abcdef123456 failed, path /key/abcdef123456')
192
+ # 'rpc *** failed, path ***'
193
+ masker = SecretsMasker(
194
+ {'ACCOUNT': '{"type": "service_account", "private_key": "-----BEGIN KEY-----\\nMIIEvQIBADANBg\\n-----END KEY-----"}'}
195
+ )
196
+ masker.mask('key MIIEvQIBADANBg of type service_account')
197
+ # 'key *** of *** ***'
198
+ ```
199
+
200
+ The value without spaces and newlines around it, and without a trailing `/`:
201
+
202
+ ```py
203
+ masker.mask('token tok_9f8e7d6c5b4a expired')
204
+ # 'token *** expired'
205
+ masker.mask('connect https://rpc.example.com/v3/a1b2c3d4e5f6')
206
+ # 'connect ***'
207
+ ```
208
+
209
+ The parts of a URL on their own: the host with its credentials, the path, the path with the query, the query, each `name=value` pair, the user name, the password and `user:password`, percent-decoded as well, and the credentials as an HTTP Basic header. HTTP libraries often log only the path:
210
+
211
+ ```py
212
+ masker.mask("HTTPSConnectionPool(host='rpc.example.com', port=443): Max retries exceeded with url: /v3/a1b2c3d4e5f6")
213
+ # "HTTPSConnectionPool(host='rpc.example.com', port=443): Max retries exceeded with url: ***"
214
+ masker.mask('GET https://user:s3cretPassw0rd@beacon.example.com/eth/v1/beacon/genesis')
215
+ # 'GET ***eth/v1/beacon/genesis'
216
+ masker.mask('auth user:s3cretPassw0rd rejected, header Basic dXNlcjpzM2NyZXRQYXNzdzByZA==')
217
+ # 'auth *** rejected, header Basic ***'
218
+ ```
219
+
220
+ A single segment of the path, a single query value and the host without its credentials, when they can be a key: 16 characters or more, or 8 or more with a digit. A short word of a URL (`eth`, `v1`, a host like `el`) is masked only as part of the whole path or URL, otherwise it would vanish from every line:
221
+
222
+ ```py
223
+ masker.mask('project a1b2c3d4e5f6 rate limited, host beacon.example.com')
224
+ # 'project *** rate limited, host ***'
225
+ masker.mask('GET /eth/v1/beacon/genesis from rpc.example.com')
226
+ # 'GET /eth/v1/beacon/genesis from rpc.example.com'
227
+ ```
228
+
229
+ Hex values in any case, with and without `0x`, and as the bytes they encode. Bytes as text, as hex and as their `repr()`:
230
+
231
+ ```py
232
+ masker.mask('key 0xabcdef0123456789 or ABCDEF0123456789 or 0XABCDEF0123456789')
233
+ # 'key *** or *** or ***'
234
+ masker.mask("raw 0xdeadbeef01020304 DEADBEEF01020304 b'\\xde\\xad\\xbe\\xef\\x01\\x02\\x03\\x04'")
235
+ # "raw *** *** b'***'"
236
+ masker.mask(repr(bytes.fromhex('ABCDEF0123456789')))
237
+ # "b'***'"
238
+ ```
239
+
240
+ The value percent-encoded (with `quote` and with `quote_plus`), percent-decoded, base64-encoded, and as the hex or the `repr()` of its bytes, as an application writes it into a URL, a header or a bytes body:
241
+
242
+ ```py
243
+ SecretsMasker({'T': 'tok/en+value=1'}).mask('k=tok%2Fen%2Bvalue%3D1 auth dG9rL2VuK3ZhbHVlPTE= body 0x746f6b2f656e2b76616c75653d31')
244
+ # 'k=*** auth *** body 0x***'
245
+ ```
246
+
247
+ A value shown with `repr()`, inside `str(dict)`, in an exception text or inside a JSON document, up to two layers deep (for example the `repr()` of a JSON document). Quotes, backslashes and newlines in the secret are not a problem, and a secret made of JSON syntax (a quote, a brace, `true`) is masked inside the strings of the line, so the line stays valid JSON:
248
+
249
+ ```py
250
+ password = 'pa"ss\\word\n123'
251
+ masker.add({'PASSWORD': password})
252
+ logger.info('login %r failed with %s', password, json.dumps({'password': password}))
253
+ # {..., "msg": "login '***' failed with {\"password\": \"***\"}"}
254
+ ```
255
+
256
+ The case of the text does not matter:
257
+
258
+ ```py
259
+ masker.mask('TOK_9F8E7D6C5B4A')
260
+ # '***'
261
+ ```
262
+
263
+ Numbers and other non-string values, by their text:
264
+
265
+ ```py
266
+ SecretsMasker({'PIN': 12345678}).mask('pin 12345678')
267
+ # 'pin ***'
268
+ ```
269
+
270
+ Two secrets that overlap in the text are masked as one:
271
+
272
+ ```py
273
+ SecretsMasker({'A': 'abcdefgh', 'B': 'efghijklmnop'}).mask('abcdefghijklmnop')
274
+ # '***'
275
+ ```
276
+
277
+ ## What is not masked
278
+
279
+ A value that is not in your secrets. The masker replaces known values, it does not search for text that looks like a secret:
280
+
281
+ ```py
282
+ masker.mask('other secret zzz-not-configured')
283
+ # 'other secret zzz-not-configured'
284
+ ```
285
+
286
+ A form of the value the masker does not know: a part of the value, a hash of it, more than two layers of escaping:
287
+
288
+ ```py
289
+ masker.mask('prefix a1b2c3')
290
+ # 'prefix a1b2c3'
291
+ ```
292
+
293
+ A short word of a URL on its own (see above).
294
+
295
+ A password re-encoded by an HTTP library with its own set of safe characters. The masker knows the password raw, percent-decoded and in the three encodings of `urllib.parse` (`quote` with and without the default safe characters, `quote_plus`). yarl, used by aiohttp, keeps `+ ! & =` raw and encodes the rest, so `p%40ss+x!y` is a form the masker does not know, and only the user name and the host of such a URL are masked. A per-character pattern for the credentials (each character raw or percent-encoded) is planned; until then, prefer a password without characters that need encoding.
296
+
297
+ Text that does not pass through the handler installed by `setup_logging`:
298
+
299
+ - `print`, a library that writes to the stream directly, and the message of `sys.exit('...')`, which the interpreter prints without calling the exception hook
300
+ - a logger with its own handler that does not propagate to the root logger (a logger without any handler that does not propagate goes to `logging.lastResort`, which `setup_logging` replaces by a masked handler)
301
+ - a handler added to the root logger after `setup_logging` (a `FileHandler`, the handler of an error tracker), and a `logging.config.dictConfig` call after it: they format the record themselves
302
+ - a child process of `multiprocessing`: with the `fork` start method, an exception in the child is printed by `multiprocessing` itself, without the hooks; with `spawn`, the child starts without `setup_logging` unless its module calls it
303
+
304
+ ## When logging itself fails
305
+
306
+ A record that cannot be formatted (wrong `%` arguments, a value `json` cannot serialize, a dict message nested deeper than the recursion limit) is still written as one masked JSON line, with the problem in the `error` field. Every line the library writes on its own behalf, instead of the record, starts with `[logmask]`:
307
+
308
+ ```py
309
+ logger.info('rpc %d %s', 'https://rpc.example.com/v3/a1b2c3d4e5f6', 'x')
310
+ # {..., "msg": "rpc %d %s", "error": "[logmask] Log record could not be formatted: TypeError('%d format: a real number is required, not str')", "args": "('***', 'x')"}
311
+ ```
312
+
313
+ The standard library would print this instead, to stderr and without masking:
314
+
315
+ ```text
316
+ --- Logging error ---
317
+ Traceback (most recent call last):
318
+ ...
319
+ TypeError: %d format: a real number is required, not str
320
+ Message: 'rpc %d %s'
321
+ Arguments: ('https://rpc.example.com/v3/a1b2c3d4e5f6', 'x')
322
+ ```
323
+
324
+ A record that cannot be written (the stream is closed, the pipe is broken) is reported by `SafeStreamHandler` without the record, where the standard handler would print the same `Message:` and `Arguments:` lines as above:
325
+
326
+ ```text
327
+ --- [logmask] Logging error: BrokenPipeError(32, 'Broken pipe') (record from app, /app/main.py:31) ---
328
+ ```
329
+
330
+ An unhandled exception goes through the logger as a CRITICAL line:
331
+
332
+ ```py
333
+ raise ConnectionError(f'Max retries exceeded with url: {RPC_URL}')
334
+ # {"name": "unhandled", "levelname": "CRITICAL", ..., "msg": "Unhandled exception. Shut down.", "error": "Traceback (most recent call last):\n ...\nConnectionError: Max retries exceeded with url: ***\n"}
335
+ ```
336
+
337
+ An exception in another thread gives the same line with `"msg": "Unhandled exception in a thread."` and a `thread` field; the process goes on, as with the default hook. An exception in `__del__` or in a callback (`sys.unraisablehook`) gives `"msg": "Unraisable exception."`.
338
+
339
+ Three cases cannot go through the logger and are written to stderr masked: a `KeyboardInterrupt` (Ctrl-C is not an error, so there is no CRITICAL line, the traceback is printed as plain text), an exception when no handler is configured (one JSON line written by the hook itself), and a failure of the hook (one line with the exception name).
340
+
341
+ ## Log format
342
+
343
+ Every line has these fields, in this order:
344
+
345
+ ```json
346
+ {"name": "app", "levelname": "INFO", "funcName": "main", "lineno": 29, "module": "main", "pathname": "/app/main.py", "timestamp": 1761600000, "msg": "Config", "value": {"RPC_URL": "***"}}
347
+ ```
348
+
349
+ - `name`, `levelname`, `funcName`, `lineno`, `module`, `pathname` - from the log record
350
+ - `timestamp` - the record time in whole seconds
351
+ - `msg` - the string message; a dict message adds its keys after `timestamp` instead. A dict key with the name of a record field (for example `name`) is written as `msg_name`
352
+ - `exc_info` - the traceback, for `logger.exception()` and `exc_info=True`
353
+ - bytes are written as `0x`-hex, dataclasses as dicts, sets as lists, other unknown values as text
354
+
355
+ ## Development
356
+
357
+ ```bash
358
+ poetry install
359
+ poetry run ruff format --check . && poetry run ruff check . && poetry run pyright logmask tests
360
+ poetry run pytest
361
+ ```
362
+
363
+ Mutation tests check that the suite notices a weaker masker. They run in CI on every push and pull request; locally:
364
+
365
+ ```bash
366
+ poetry run pip install "mutmut==3.8.1"
367
+ poetry run mutmut run
368
+ poetry run mutmut results # the survivors
369
+ poetry run mutmut show <mutant> # the diff of one survivor
370
+ ```
371
+
372
+ About 50 mutants survive by design (the spelling of `utf-8`, the wording of messages, the references kept by the cycle guard); CI fails above that number.
373
+
374
+ ## Release
375
+
376
+ 1. Set the version in `pyproject.toml` and move the `Unreleased` section of `CHANGELOG.md` under the version and the date.
377
+ 2. Merge to `main` and create the tag `vX.Y.Z`.
378
+ 3. Consumers update the tag in their `pyproject.toml` and run `poetry lock`.
379
+
@@ -0,0 +1,13 @@
1
+ logmask/__init__.py,sha256=UlxYUfyJKYTg1xsvyht2UDGLSxoq4kuP_a88emAWgPA,495
2
+ logmask/config.py,sha256=yT6bMassP-nVaIoPv4LsBlUME1dOdr6LsDLCCvX57EM,1289
3
+ logmask/configure.py,sha256=HBVX3m6lNf3_8mofkfDamsCE5d9llbMTcMj4KhN0AHA,2444
4
+ logmask/excepthook.py,sha256=bqbsJcZbXtBwbcp0W2hWm1FUEZne0y41kCUmgbNDmtk,4831
5
+ logmask/formatter.py,sha256=RoCY3DNmD232oen27bLjtZ2jCKkIx4YlBAPsrad14xQ,4672
6
+ logmask/handler.py,sha256=VWXgv8vcIGvLzB1dRhO99fmRXeSGDSz_lzwcibni9WE,1163
7
+ logmask/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
+ logmask/secrets.py,sha256=aStieUQ-U6H-xkmNdCfxAhUaK4EaRFABwSfaQzpSIr8,13293
9
+ logmask/structlog_processor.py,sha256=2xhxFbqaBUYa-SQPEtiFZgbBt6KrcLiG30735U190Xc,1298
10
+ logmask-0.1.3.dist-info/METADATA,sha256=DOW4BgTTH6w-3SD3my9hqCr4UY0dBcc2zKs9k7czOjM,18594
11
+ logmask-0.1.3.dist-info/WHEEL,sha256=L-WvLdSBvJWHXv5zrQxDvNRnIgoBzLUpcbuiw19KCuQ,88
12
+ logmask-0.1.3.dist-info/licenses/LICENSE,sha256=LpEi6KieJAlefSgmC1tWdhIUzPCoAis2DEZRzQbD6e0,1061
13
+ logmask-0.1.3.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: poetry-core 2.5.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Lido
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.