logmask 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- logmask/__init__.py +18 -0
- logmask/config.py +29 -0
- logmask/configure.py +51 -0
- logmask/excepthook.py +102 -0
- logmask/formatter.py +107 -0
- logmask/handler.py +28 -0
- logmask/py.typed +0 -0
- logmask/secrets.py +302 -0
- logmask/structlog_processor.py +27 -0
- logmask-0.1.3.dist-info/METADATA +379 -0
- logmask-0.1.3.dist-info/RECORD +13 -0
- logmask-0.1.3.dist-info/WHEEL +4 -0
- logmask-0.1.3.dist-info/licenses/LICENSE +21 -0
logmask/__init__.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from .config import masked_config
|
|
2
|
+
from .configure import setup_logging
|
|
3
|
+
from .excepthook import install_excepthook
|
|
4
|
+
from .formatter import JsonFormatter
|
|
5
|
+
from .handler import SafeStreamHandler
|
|
6
|
+
from .secrets import MASKED_VALUE, SecretsMasker
|
|
7
|
+
from .structlog_processor import mask_secrets_processor
|
|
8
|
+
|
|
9
|
+
__all__ = (
|
|
10
|
+
'MASKED_VALUE',
|
|
11
|
+
'JsonFormatter',
|
|
12
|
+
'SafeStreamHandler',
|
|
13
|
+
'SecretsMasker',
|
|
14
|
+
'install_excepthook',
|
|
15
|
+
'mask_secrets_processor',
|
|
16
|
+
'masked_config',
|
|
17
|
+
'setup_logging',
|
|
18
|
+
)
|
logmask/config.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from collections.abc import Mapping
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
from .secrets import MASKED_VALUE, check_private_value
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def masked_config(public: Mapping[str, Any], private: Mapping[str, Any]) -> dict[str, Any]:
|
|
8
|
+
"""The config for the startup log: public values as they are, private values replaced by the placeholder.
|
|
9
|
+
|
|
10
|
+
A private list keeps its length, and an unset private value (None or an empty string) becomes an empty string, so
|
|
11
|
+
the log shows which variables are set without reading their values. Any other value, `0` and `False` included, is
|
|
12
|
+
set and shows the placeholder. A variable in both mappings is an error, and so is a private value of a type the
|
|
13
|
+
masker cannot search for secrets, the same error `setup_logging` raises.
|
|
14
|
+
"""
|
|
15
|
+
overlap = sorted(set(public) & set(private))
|
|
16
|
+
if overlap:
|
|
17
|
+
raise ValueError(f'Variables listed as both public and private: {overlap}')
|
|
18
|
+
for value in private.values():
|
|
19
|
+
check_private_value(value)
|
|
20
|
+
|
|
21
|
+
return {**public, **{key: _mask(value) for key, value in private.items()}}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _mask(value: Any) -> Any:
|
|
25
|
+
if value is None or (isinstance(value, str | bytes) and not value):
|
|
26
|
+
return ''
|
|
27
|
+
if isinstance(value, list | tuple):
|
|
28
|
+
return [_mask(item) for item in value]
|
|
29
|
+
return MASKED_VALUE
|
logmask/configure.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import threading
|
|
3
|
+
from collections.abc import Mapping
|
|
4
|
+
from typing import IO
|
|
5
|
+
|
|
6
|
+
from .excepthook import install_excepthook
|
|
7
|
+
from .formatter import JsonFormatter
|
|
8
|
+
from .handler import SafeStreamHandler
|
|
9
|
+
from .secrets import SecretsMasker
|
|
10
|
+
|
|
11
|
+
# One masker per process: a second call adds its secrets to the same masker, so the secrets of the first call stay
|
|
12
|
+
# masked after a library or a test configures logging again.
|
|
13
|
+
_masker: SecretsMasker | None = None
|
|
14
|
+
_lock = threading.Lock()
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def setup_logging(secrets: Mapping[str, object], *, level: int | str = logging.INFO, stream: IO[str] | None = None) -> SecretsMasker:
|
|
18
|
+
"""Configure the root logger: JSON lines to the stream (stderr by default), every line masked.
|
|
19
|
+
|
|
20
|
+
`secrets` is the mapping of private variables; a service that reads its secrets later passes `{}` and calls
|
|
21
|
+
`add()` on the returned masker. A private value of a type the masker cannot search raises `TypeError`. Existing
|
|
22
|
+
handlers of the root logger are replaced, the handler of last resort (for a logger that does not propagate and
|
|
23
|
+
has no handler) is replaced by a masked one, warnings are routed through the logger, and the exception hooks of
|
|
24
|
+
the main thread, of the other threads and of the unraisable exceptions are installed, so every traceback is
|
|
25
|
+
masked too. The same masker serves a structlog processor or masks text by hand. `level` is a logging level or its
|
|
26
|
+
name in any case.
|
|
27
|
+
"""
|
|
28
|
+
global _masker
|
|
29
|
+
with _lock:
|
|
30
|
+
if _masker is None:
|
|
31
|
+
_masker = SecretsMasker(secrets)
|
|
32
|
+
else:
|
|
33
|
+
_masker.add(secrets)
|
|
34
|
+
masker = _masker
|
|
35
|
+
if isinstance(level, str):
|
|
36
|
+
level = level.upper()
|
|
37
|
+
|
|
38
|
+
handler = SafeStreamHandler(stream, masker=masker)
|
|
39
|
+
handler.setFormatter(JsonFormatter(masker))
|
|
40
|
+
logging.basicConfig(level=level, handlers=[handler], force=True)
|
|
41
|
+
# A logger with propagate=False and no handler of its own goes to logging.lastResort, which writes the raw
|
|
42
|
+
# message to stderr. The replacement writes a masked JSON line to the same stream, at the same level (WARNING).
|
|
43
|
+
last_resort = SafeStreamHandler(stream, masker=masker)
|
|
44
|
+
last_resort.setFormatter(JsonFormatter(masker))
|
|
45
|
+
last_resort.setLevel(logging.WARNING)
|
|
46
|
+
logging.lastResort = last_resort
|
|
47
|
+
# warnings.warn() prints to stderr by default; a warning text may carry a URL.
|
|
48
|
+
logging.captureWarnings(True)
|
|
49
|
+
|
|
50
|
+
install_excepthook(masker=masker)
|
|
51
|
+
return masker
|
logmask/excepthook.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# sys.UnraisableHookArgs exists only in the type stubs: the annotations must not be evaluated at runtime.
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import contextlib
|
|
5
|
+
import logging
|
|
6
|
+
import sys
|
|
7
|
+
import threading
|
|
8
|
+
import traceback
|
|
9
|
+
from types import TracebackType
|
|
10
|
+
|
|
11
|
+
from .formatter import JsonFormatter
|
|
12
|
+
from .handler import SafeStreamHandler
|
|
13
|
+
from .secrets import LIBRARY_TAG, SecretsMasker
|
|
14
|
+
|
|
15
|
+
LOGGER_NAME = 'unhandled'
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _format(exc_type: type[BaseException], exc_value: BaseException | None, exc_traceback: TracebackType | None) -> str:
|
|
19
|
+
if exc_value is None:
|
|
20
|
+
return f'{exc_type.__name__}\n' + ''.join(traceback.format_tb(exc_traceback))
|
|
21
|
+
return ''.join(traceback.format_exception(exc_type, exc_value, exc_traceback))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _log_exception(
|
|
25
|
+
exc_type: type[BaseException],
|
|
26
|
+
exc_value: BaseException | None,
|
|
27
|
+
exc_traceback: TracebackType | None,
|
|
28
|
+
*,
|
|
29
|
+
masker: SecretsMasker,
|
|
30
|
+
message: str,
|
|
31
|
+
thread_name: str | None = None,
|
|
32
|
+
context: str | None = None,
|
|
33
|
+
) -> None:
|
|
34
|
+
"""Send the traceback through the logger, so that it is formatted and masked too.
|
|
35
|
+
|
|
36
|
+
A stop by Ctrl-C is not an error: for KeyboardInterrupt the traceback goes to stderr as plain text, like the
|
|
37
|
+
default hook prints it, but masked. The chained exceptions in it (`__context__`) may carry a URL or a key.
|
|
38
|
+
The traceback text is masked before it enters the record: a handler that is not the masking one (added by a
|
|
39
|
+
library after the setup) would otherwise print it raw. When the logger has no handler, the line is written to
|
|
40
|
+
stderr by this module, masked, instead of the raw record that `logging.lastResort` would print. When anything in
|
|
41
|
+
here fails, one masked line goes to stderr: the interpreter would otherwise print the hook's error together with
|
|
42
|
+
the original traceback, unmasked.
|
|
43
|
+
"""
|
|
44
|
+
try:
|
|
45
|
+
error = masker.mask(_format(exc_type, exc_value, exc_traceback))
|
|
46
|
+
if issubclass(exc_type, KeyboardInterrupt):
|
|
47
|
+
sys.stderr.write(error)
|
|
48
|
+
return
|
|
49
|
+
|
|
50
|
+
record_message: dict[str, object] = {'msg': message, 'error': error}
|
|
51
|
+
if thread_name is not None:
|
|
52
|
+
record_message['thread'] = masker.mask(thread_name)
|
|
53
|
+
if context is not None:
|
|
54
|
+
record_message['context'] = masker.mask(context)
|
|
55
|
+
|
|
56
|
+
logger = logging.getLogger(LOGGER_NAME)
|
|
57
|
+
if logger.hasHandlers():
|
|
58
|
+
logger.critical(record_message)
|
|
59
|
+
return
|
|
60
|
+
record = logger.makeRecord(logger.name, logging.CRITICAL, '(unknown file)', 0, record_message, (), None)
|
|
61
|
+
handler = SafeStreamHandler(sys.stderr, masker=masker)
|
|
62
|
+
handler.setFormatter(JsonFormatter(masker))
|
|
63
|
+
handler.handle(record)
|
|
64
|
+
except BaseException as failure: # noqa: BLE001 - the hook must not raise, see the docstring
|
|
65
|
+
with contextlib.suppress(Exception):
|
|
66
|
+
sys.stderr.write(masker.mask(f'--- {LIBRARY_TAG} {exc_type.__name__} could not be logged: {type(failure).__name__} ---\n'))
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def install_excepthook(*, masker: SecretsMasker) -> None:
|
|
70
|
+
"""Route the unhandled exceptions of the main thread, of the other threads, and the unraisable ones through the logger.
|
|
71
|
+
|
|
72
|
+
`masker` covers the output that cannot go through the logger: a KeyboardInterrupt traceback, any exception when
|
|
73
|
+
no handler is configured, and a failure of the hook itself. The logger's own handler masks the rest.
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
def hook(exc_type: type[BaseException], exc_value: BaseException, exc_traceback: TracebackType | None) -> None:
|
|
77
|
+
_log_exception(exc_type, exc_value, exc_traceback, masker=masker, message='Unhandled exception. Shut down.')
|
|
78
|
+
|
|
79
|
+
def thread_hook(args: threading.ExceptHookArgs) -> None:
|
|
80
|
+
if issubclass(args.exc_type, SystemExit):
|
|
81
|
+
# The default thread hook ignores sys.exit() in a thread; so do we.
|
|
82
|
+
return
|
|
83
|
+
thread_name = args.thread.name if args.thread is not None else None
|
|
84
|
+
_log_exception(
|
|
85
|
+
args.exc_type,
|
|
86
|
+
args.exc_value,
|
|
87
|
+
args.exc_traceback,
|
|
88
|
+
masker=masker,
|
|
89
|
+
message='Unhandled exception in a thread.',
|
|
90
|
+
thread_name=thread_name,
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
def unraisable_hook(args: sys.UnraisableHookArgs) -> None:
|
|
94
|
+
# An exception in __del__, in a weakref callback or in a GC hook: the default hook prints it to stderr.
|
|
95
|
+
# err_msg says where, when the interpreter sets it (Python 3.14 and later); otherwise the type of the object
|
|
96
|
+
# stands in. The object itself is not logged, its repr is unknown text.
|
|
97
|
+
context = args.err_msg or f'Exception ignored in: {type(args.object).__name__}'
|
|
98
|
+
_log_exception(args.exc_type, args.exc_value, args.exc_traceback, masker=masker, message='Unraisable exception.', context=context)
|
|
99
|
+
|
|
100
|
+
sys.excepthook = hook
|
|
101
|
+
threading.excepthook = thread_hook
|
|
102
|
+
sys.unraisablehook = unraisable_hook
|
logmask/formatter.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import dataclasses
|
|
2
|
+
import json
|
|
3
|
+
import logging
|
|
4
|
+
from collections.abc import Callable, Iterator, Mapping
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from .secrets import LIBRARY_TAG, SecretsMasker
|
|
8
|
+
|
|
9
|
+
UNPRINTABLE = '<unprintable>'
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def json_default(value: object) -> Any:
|
|
13
|
+
"""Values json.dumps does not know: bytes as 0x-hex, dataclasses as dicts, sets and iterators as lists, the rest as text."""
|
|
14
|
+
if isinstance(value, bytes | bytearray):
|
|
15
|
+
return '0x' + bytes(value).hex()
|
|
16
|
+
if dataclasses.is_dataclass(value) and not isinstance(value, type):
|
|
17
|
+
return dataclasses.asdict(value)
|
|
18
|
+
if isinstance(value, Mapping):
|
|
19
|
+
return dict(value)
|
|
20
|
+
if isinstance(value, set | frozenset | Iterator):
|
|
21
|
+
return list(value)
|
|
22
|
+
return str(value)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _safe(convert: Callable[[Any], str], value: Any) -> str:
|
|
26
|
+
try:
|
|
27
|
+
return convert(value)
|
|
28
|
+
except Exception: # noqa: BLE001 - a __str__ or __repr__ that raises must not break the fallback line
|
|
29
|
+
return UNPRINTABLE
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class JsonFormatter(logging.Formatter):
|
|
33
|
+
"""One JSON object per line with a fixed set of record fields; every line goes through the masker.
|
|
34
|
+
|
|
35
|
+
`format()` never raises: a record that cannot be formatted becomes a line with an `error` field. Otherwise the
|
|
36
|
+
handler would report the error with the raw message and arguments, without masking, or the error would reach the
|
|
37
|
+
caller of the logger.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
def __init__(self, masker: SecretsMasker):
|
|
41
|
+
super().__init__()
|
|
42
|
+
self._masker = masker
|
|
43
|
+
|
|
44
|
+
def format(self, record: logging.LogRecord) -> str:
|
|
45
|
+
try:
|
|
46
|
+
text = json.dumps(self._line(record), default=json_default)
|
|
47
|
+
except Exception as error: # noqa: BLE001 - any formatting error must become a masked line, see the class docstring
|
|
48
|
+
text = json.dumps(self._fallback_line(record, error))
|
|
49
|
+
# Secrets are hidden in every log line, including messages from third-party libraries.
|
|
50
|
+
masked = self._masker.mask(text)
|
|
51
|
+
try:
|
|
52
|
+
json.loads(masked)
|
|
53
|
+
return masked
|
|
54
|
+
except ValueError:
|
|
55
|
+
pass
|
|
56
|
+
# A secret made of JSON syntax (a quote, a brace, `true`, a number) was replaced outside the strings. Mask
|
|
57
|
+
# the strings one by one instead; the JSON-escaped forms are then not needed, and the syntax stays.
|
|
58
|
+
try:
|
|
59
|
+
return json.dumps(self._mask_leaves(json.loads(text)))
|
|
60
|
+
except Exception as error: # noqa: BLE001 - a message nested too deep for the recursion limit, see the class docstring
|
|
61
|
+
# The fallback line is flat: its values are the record fields and short texts.
|
|
62
|
+
return json.dumps(self._mask_leaves(self._fallback_line(record, error)))
|
|
63
|
+
|
|
64
|
+
def _mask_leaves(self, value: Any) -> Any:
|
|
65
|
+
if isinstance(value, str):
|
|
66
|
+
return self._masker.mask(value)
|
|
67
|
+
if isinstance(value, dict):
|
|
68
|
+
return {self._masker.mask(key): self._mask_leaves(item) for key, item in value.items()}
|
|
69
|
+
if isinstance(value, list):
|
|
70
|
+
return [self._mask_leaves(item) for item in value]
|
|
71
|
+
if isinstance(value, int | float) and not isinstance(value, bool):
|
|
72
|
+
# A number equal to a secret (a PIN, an id) is what broke the line: it is replaced by the masked text.
|
|
73
|
+
text = str(value)
|
|
74
|
+
masked = self._masker.mask(text)
|
|
75
|
+
return value if masked == text else masked
|
|
76
|
+
return value
|
|
77
|
+
|
|
78
|
+
def _base(self, record: logging.LogRecord) -> dict[str, Any]:
|
|
79
|
+
return {
|
|
80
|
+
'name': record.name,
|
|
81
|
+
'levelname': record.levelname,
|
|
82
|
+
'funcName': record.funcName,
|
|
83
|
+
'lineno': record.lineno,
|
|
84
|
+
'module': record.module,
|
|
85
|
+
'pathname': record.pathname,
|
|
86
|
+
'timestamp': int(record.created),
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
def _line(self, record: logging.LogRecord) -> dict[str, Any]:
|
|
90
|
+
message = record.msg if isinstance(record.msg, dict) else {'msg': record.getMessage()}
|
|
91
|
+
line = self._base(record)
|
|
92
|
+
for key, value in message.items():
|
|
93
|
+
# The record fields win: a message key `name` or `timestamp` would break every query on them.
|
|
94
|
+
line[f'msg_{key}' if key in line else key] = value
|
|
95
|
+
if record.exc_info:
|
|
96
|
+
line['exc_info'] = self.formatException(record.exc_info)
|
|
97
|
+
return line
|
|
98
|
+
|
|
99
|
+
def _fallback_line(self, record: logging.LogRecord, error: Exception) -> dict[str, Any]:
|
|
100
|
+
line = {
|
|
101
|
+
**self._base(record),
|
|
102
|
+
'msg': _safe(str, record.msg),
|
|
103
|
+
'error': f'{LIBRARY_TAG} Log record could not be formatted: {_safe(repr, error)}',
|
|
104
|
+
}
|
|
105
|
+
if record.args:
|
|
106
|
+
line['args'] = _safe(repr, record.args)
|
|
107
|
+
return line
|
logmask/handler.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import contextlib
|
|
2
|
+
import logging
|
|
3
|
+
import sys
|
|
4
|
+
from typing import IO
|
|
5
|
+
|
|
6
|
+
from .secrets import LIBRARY_TAG, SecretsMasker
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class SafeStreamHandler(logging.StreamHandler): # type: ignore[type-arg]
|
|
10
|
+
"""A StreamHandler whose error report does not show the record.
|
|
11
|
+
|
|
12
|
+
When a handler cannot write a record (a closed stream, a broken pipe), the standard `handleError` prints the raw
|
|
13
|
+
message and the arguments to stderr, without masking. This handler reports the error and where the record came
|
|
14
|
+
from, masked, and nothing else from the record.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
def __init__(self, stream: IO[str] | None = None, *, masker: SecretsMasker):
|
|
18
|
+
super().__init__(stream)
|
|
19
|
+
self._masker = masker
|
|
20
|
+
|
|
21
|
+
def handleError(self, record: logging.LogRecord) -> None:
|
|
22
|
+
if not logging.raiseExceptions:
|
|
23
|
+
return
|
|
24
|
+
error = sys.exc_info()[1]
|
|
25
|
+
text = f'--- {LIBRARY_TAG} Logging error: {error!r} (record from {record.name}, {record.pathname}:{record.lineno}) ---\n'
|
|
26
|
+
# stderr may be the stream that failed; there is nowhere else to report.
|
|
27
|
+
with contextlib.suppress(Exception):
|
|
28
|
+
sys.stderr.write(self._masker.mask(text))
|
logmask/py.typed
ADDED
|
File without changes
|
logmask/secrets.py
ADDED
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
import dataclasses
|
|
3
|
+
import json
|
|
4
|
+
import numbers
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
import threading
|
|
8
|
+
from collections.abc import Iterable, Iterator, Mapping
|
|
9
|
+
from urllib.parse import quote, quote_plus, unquote, unquote_plus, urlsplit
|
|
10
|
+
|
|
11
|
+
MASKED_VALUE = '***'
|
|
12
|
+
# Every line the library writes on its own behalf, instead of the record, starts with this tag.
|
|
13
|
+
LIBRARY_TAG = '[logmask]'
|
|
14
|
+
# Every private value is masked whole, whatever its length. A standalone fragment of a private URL (the host, one
|
|
15
|
+
# path segment, one query value) is masked on its own only when it can be a key: 16 characters or more, or 8 or more
|
|
16
|
+
# with a digit. The whole URL, the whole path, the whole query and the user:password are masked without any
|
|
17
|
+
# condition. Masking a word like `eth`, a segment like `v3` or a host like `el` on its own would erase it from every
|
|
18
|
+
# log line.
|
|
19
|
+
_KEY_FRAGMENT_LENGTH = 16
|
|
20
|
+
_KEY_FRAGMENT_LENGTH_WITH_DIGIT = 8
|
|
21
|
+
# The encodings of a value (percent, base64, hex of its bytes) are derived for values of 4 characters or more: the
|
|
22
|
+
# encodings of a shorter value are short random-looking strings that hit ordinary words. The value itself is always
|
|
23
|
+
# masked.
|
|
24
|
+
_ENCODED_FORMS_MIN_LENGTH = 4
|
|
25
|
+
# How many layers of repr() or json.dumps() around a secret are recognised, on top of the JSON encoding of the line.
|
|
26
|
+
_ESCAPE_DEPTH = 2
|
|
27
|
+
|
|
28
|
+
_HEX = re.compile(r'(0x)?([0-9a-f]+)', re.ASCII | re.IGNORECASE)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _key_fragment(text: str) -> bool:
|
|
32
|
+
return len(text) >= _KEY_FRAGMENT_LENGTH or (len(text) >= _KEY_FRAGMENT_LENGTH_WITH_DIGIT and any(char.isdigit() for char in text))
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _utf8(text: str) -> bytes:
|
|
36
|
+
# A value read from the environment may carry bytes that are not UTF-8, as lone surrogates (surrogateescape).
|
|
37
|
+
return text.encode('utf-8', errors='surrogateescape')
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _hex_forms(text: str) -> Iterator[str]:
|
|
41
|
+
# A hex secret is logged with or without the 0x prefix, and as the bytes it encodes (a private key is held as
|
|
42
|
+
# bytes by eth_account); the pattern ignores the case.
|
|
43
|
+
match = _HEX.fullmatch(text)
|
|
44
|
+
if match is None:
|
|
45
|
+
return
|
|
46
|
+
digits = match[2]
|
|
47
|
+
yield digits
|
|
48
|
+
yield '0x' + digits
|
|
49
|
+
if len(digits) % 2 == 0:
|
|
50
|
+
yield repr(bytes.fromhex(digits))[2:-1]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _encoded_forms(text: str) -> Iterator[str]:
|
|
54
|
+
"""The forms an application gives a raw secret when it builds a URL, a header or a bytes value from it."""
|
|
55
|
+
if len(text) < _ENCODED_FORMS_MIN_LENGTH:
|
|
56
|
+
return
|
|
57
|
+
raw = _utf8(text)
|
|
58
|
+
yield quote(text, safe='', errors='surrogateescape')
|
|
59
|
+
yield quote(text, errors='surrogateescape')
|
|
60
|
+
yield quote_plus(text, errors='surrogateescape')
|
|
61
|
+
for encoded in (base64.b64encode(raw), base64.urlsafe_b64encode(raw)):
|
|
62
|
+
yield encoded.decode()
|
|
63
|
+
yield encoded.decode().rstrip('=')
|
|
64
|
+
# json_default writes a bytes value as 0x-hex; a text secret inside such a value must be found in that form too.
|
|
65
|
+
yield raw.hex()
|
|
66
|
+
yield repr(raw)[2:-1]
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _with_decoded(text: str) -> Iterator[str]:
|
|
70
|
+
yield text
|
|
71
|
+
yield unquote(text)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _url_parts(text: str) -> Iterator[str]:
|
|
75
|
+
try:
|
|
76
|
+
url = urlsplit(text)
|
|
77
|
+
except ValueError:
|
|
78
|
+
return
|
|
79
|
+
if not (url.scheme and url.netloc):
|
|
80
|
+
return
|
|
81
|
+
|
|
82
|
+
# The netloc with the credentials always, and the host alone when a provider may have put the key into it.
|
|
83
|
+
if '@' in url.netloc:
|
|
84
|
+
yield url.netloc
|
|
85
|
+
for host in (url.netloc.rpartition('@')[2], url.hostname or ''):
|
|
86
|
+
if _key_fragment(host):
|
|
87
|
+
yield host
|
|
88
|
+
for part in (url.username, url.password):
|
|
89
|
+
if part:
|
|
90
|
+
yield from _with_decoded(part)
|
|
91
|
+
if url.password:
|
|
92
|
+
userinfo = f'{url.username or ""}:{url.password}'
|
|
93
|
+
yield from _with_decoded(userinfo)
|
|
94
|
+
# The same credentials as an HTTP Basic header.
|
|
95
|
+
yield base64.b64encode(_utf8(unquote(userinfo))).decode()
|
|
96
|
+
|
|
97
|
+
# HTTP libraries log the path, or the path with the query, without the host.
|
|
98
|
+
if url.path.strip('/'):
|
|
99
|
+
yield from _with_decoded(url.path)
|
|
100
|
+
yield url.path.rstrip('/')
|
|
101
|
+
for segment in url.path.split('/'):
|
|
102
|
+
if _key_fragment(segment):
|
|
103
|
+
yield from _with_decoded(segment)
|
|
104
|
+
if url.query:
|
|
105
|
+
yield url.query
|
|
106
|
+
yield f'{url.path}?{url.query}'
|
|
107
|
+
for pair in url.query.split('&'):
|
|
108
|
+
yield pair
|
|
109
|
+
value = pair.partition('=')[2]
|
|
110
|
+
if _key_fragment(value):
|
|
111
|
+
yield from _with_decoded(value)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _variants(text: str) -> Iterator[str]:
|
|
115
|
+
# A secret read from a file usually keeps the trailing newline, the log shows the stripped value.
|
|
116
|
+
for form in dict.fromkeys((text, text.strip())):
|
|
117
|
+
yield form
|
|
118
|
+
yield form.rstrip('/')
|
|
119
|
+
# A value stored percent-encoded (a password inside a URL) is used decoded by the application.
|
|
120
|
+
yield unquote(form)
|
|
121
|
+
yield unquote_plus(form)
|
|
122
|
+
yield from _hex_forms(form)
|
|
123
|
+
yield from _url_parts(form)
|
|
124
|
+
yield from _encoded_forms(form)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _encodings(text: str) -> Iterator[str]:
|
|
128
|
+
"""The body of a string after one layer of repr() or json.dumps()."""
|
|
129
|
+
# repr() picks the quote style from the whole string: a secret with a ' inside a string that also has a " is
|
|
130
|
+
# escaped as \'. Both bodies are needed.
|
|
131
|
+
yield repr(text)[1:-1]
|
|
132
|
+
yield repr(text + '"')[1:-2]
|
|
133
|
+
yield json.dumps(text)[1:-1]
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _escaped_forms(text: str) -> set[str]:
|
|
137
|
+
forms = {text}
|
|
138
|
+
layer = {text}
|
|
139
|
+
for _ in range(_ESCAPE_DEPTH):
|
|
140
|
+
layer = {encoded for form in layer for encoded in _encodings(form)}
|
|
141
|
+
forms |= layer
|
|
142
|
+
return forms
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
SUPPORTED_TYPES = 'str, bytes, a number, a path, a mapping, a list or another iterable, a dataclass or an object with get_secret_value()'
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _strings_inside(text: str, seen: dict[int, object]) -> Iterator[str]:
|
|
149
|
+
"""The values held by one string: a JSON document (a list of URLs, the JSON of a service account) is searched as
|
|
150
|
+
the container it encodes, several URLs separated by commas, semicolons or white space are searched one by one
|
|
151
|
+
(otherwise only the last URL would get its parts, the others would be plain prefixes of the value), and every
|
|
152
|
+
line of a multi-line value (a PEM key) is a secret on its own, as a library prints it line by line."""
|
|
153
|
+
if text.lstrip()[:1] in ('[', '{'):
|
|
154
|
+
try:
|
|
155
|
+
document = json.loads(text)
|
|
156
|
+
except ValueError:
|
|
157
|
+
document = None
|
|
158
|
+
if isinstance(document, list | dict):
|
|
159
|
+
yield from _strings(document, seen)
|
|
160
|
+
return
|
|
161
|
+
urls = [piece for piece in re.split(r'[,;\s]+', text) if '://' in piece]
|
|
162
|
+
if len(urls) > 1:
|
|
163
|
+
yield from urls
|
|
164
|
+
lines = [line.strip() for line in text.splitlines()]
|
|
165
|
+
if len(lines) > 1:
|
|
166
|
+
yield from (line for line in lines if line)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _strings(value: object, seen: dict[int, object]) -> Iterator[str]:
|
|
170
|
+
"""Every string inside the value: strings and the values a string holds (a JSON document, a list of URLs), bytes
|
|
171
|
+
in their text, hex and repr forms, the items of containers and other iterables (the keys of a nested mapping are
|
|
172
|
+
data, like its values; the keys of the top-level mapping are the variable names and never reach here), the fields
|
|
173
|
+
of a dataclass, the wrapped value of a SecretStr-like object, numbers and paths as text. None and bool are not
|
|
174
|
+
secrets. Any other type is refused: the masker cannot promise that it found every secret inside it.
|
|
175
|
+
|
|
176
|
+
`seen` keeps a reference to every container visited, so that a cycle stops and a freed temporary cannot hand its
|
|
177
|
+
id to a later container that would then be skipped.
|
|
178
|
+
"""
|
|
179
|
+
if value is None or isinstance(value, bool):
|
|
180
|
+
return
|
|
181
|
+
if isinstance(value, str):
|
|
182
|
+
yield value
|
|
183
|
+
yield from _strings_inside(value, seen)
|
|
184
|
+
elif isinstance(value, bytes | bytearray | memoryview):
|
|
185
|
+
raw = bytes(value)
|
|
186
|
+
yield raw.decode('utf-8', errors='surrogateescape')
|
|
187
|
+
yield raw.hex()
|
|
188
|
+
yield repr(raw)[2:-1]
|
|
189
|
+
elif isinstance(value, numbers.Number | os.PathLike):
|
|
190
|
+
# Whatever text the value shows in a log is a secret too.
|
|
191
|
+
yield str(value)
|
|
192
|
+
elif id(value) in seen:
|
|
193
|
+
return
|
|
194
|
+
elif isinstance(value, Mapping):
|
|
195
|
+
seen[id(value)] = value
|
|
196
|
+
for key, item in value.items():
|
|
197
|
+
yield from _strings(key, seen)
|
|
198
|
+
yield from _strings(item, seen)
|
|
199
|
+
elif isinstance(value, Iterable):
|
|
200
|
+
seen[id(value)] = value
|
|
201
|
+
for item in value:
|
|
202
|
+
yield from _strings(item, seen)
|
|
203
|
+
elif callable(getter := getattr(value, 'get_secret_value', None)):
|
|
204
|
+
# pydantic SecretStr and the like: str() shows a placeholder, the secret is behind the method.
|
|
205
|
+
seen[id(value)] = value
|
|
206
|
+
yield from _strings(getter(), seen)
|
|
207
|
+
elif dataclasses.is_dataclass(value):
|
|
208
|
+
seen[id(value)] = value
|
|
209
|
+
for field in dataclasses.fields(value):
|
|
210
|
+
yield from _strings(getattr(value, field.name), seen)
|
|
211
|
+
else:
|
|
212
|
+
raise TypeError(f'A private value of type {type(value).__name__} is not supported, pass {SUPPORTED_TYPES}')
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def check_private_value(value: object) -> None:
|
|
216
|
+
"""Raise `TypeError` for a value the masker cannot search for secrets, by the rule of `SecretsMasker`."""
|
|
217
|
+
for _ in _strings(value, {}):
|
|
218
|
+
pass
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _secret_values(secrets: Mapping[str, object]) -> frozenset[str]:
|
|
222
|
+
"""Every string to hide in the formatted log line, derived from the secret values."""
|
|
223
|
+
forms: set[str] = set()
|
|
224
|
+
for value in secrets.values():
|
|
225
|
+
for item in _strings(value, {}):
|
|
226
|
+
for variant in _variants(item):
|
|
227
|
+
if variant:
|
|
228
|
+
forms |= _escaped_forms(variant)
|
|
229
|
+
# Every form once more JSON-escaped: the replacement runs on the formatted JSON line.
|
|
230
|
+
return frozenset(forms | {json.dumps(form)[1:-1] for form in forms})
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _compile(forms: Iterable[str], flags: int) -> re.Pattern[str]:
|
|
234
|
+
# Longest first, so that a value which contains another one is replaced as a whole. The alternation tries the
|
|
235
|
+
# values in order, and the lookahead makes the search report a match at every position, also inside an earlier
|
|
236
|
+
# match: overlapping secrets are merged into one span, a plain substitution would leave the tail of the second
|
|
237
|
+
# one visible.
|
|
238
|
+
ordered = sorted(set(forms), key=lambda form: (-len(form), form))
|
|
239
|
+
pattern = re.compile('(?=(' + '|'.join(map(re.escape, ordered)) + '))', flags)
|
|
240
|
+
# re keeps the last few hundred compiled patterns in a process-wide cache, keyed by their source. Every add()
|
|
241
|
+
# compiles a bigger pattern, and the earlier ones, with the text of the secrets, would stay in that cache for
|
|
242
|
+
# the life of the process: hundreds of MiB after a few hundred token rotations, and secrets that outlive the
|
|
243
|
+
# masker. The cache is dropped; other modules only recompile their patterns once.
|
|
244
|
+
re.purge()
|
|
245
|
+
return pattern
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
class SecretsMasker:
|
|
249
|
+
"""Replaces every form of the secret values with the masked value in any text, for example in a formatted log line.
|
|
250
|
+
|
|
251
|
+
The case of the text does not matter: a key is logged upper-case as often as lower-case. New secrets can be added
|
|
252
|
+
later with `add()`, for a service that configures logging before it reads its config. A value of a type the masker
|
|
253
|
+
cannot search (`SUPPORTED_TYPES` lists the supported ones) raises `TypeError`, in the constructor and in `add()`.
|
|
254
|
+
"""
|
|
255
|
+
|
|
256
|
+
def __init__(self, secrets: Mapping[str, object]):
|
|
257
|
+
self._lock = threading.Lock()
|
|
258
|
+
self._forms: frozenset[str] = frozenset()
|
|
259
|
+
# One pattern for ASCII text, matched on the lower-cased text (same offsets, half the cost), and one
|
|
260
|
+
# case-insensitive pattern for text whose lower case may change the offsets.
|
|
261
|
+
self._pattern_ascii: re.Pattern[str] | None = None
|
|
262
|
+
self._pattern: re.Pattern[str] | None = None
|
|
263
|
+
self.add(secrets)
|
|
264
|
+
|
|
265
|
+
def add(self, secrets: Mapping[str, object]) -> None:
|
|
266
|
+
"""Mask the values of these secrets too, in every line from now on."""
|
|
267
|
+
new_forms = _secret_values(secrets)
|
|
268
|
+
with self._lock:
|
|
269
|
+
forms = self._forms | new_forms
|
|
270
|
+
if forms == self._forms:
|
|
271
|
+
return
|
|
272
|
+
pattern_ascii = _compile((form.lower() for form in forms), 0)
|
|
273
|
+
pattern = _compile(forms, re.IGNORECASE)
|
|
274
|
+
self._forms, self._pattern_ascii, self._pattern = forms, pattern_ascii, pattern
|
|
275
|
+
|
|
276
|
+
def __repr__(self) -> str:
|
|
277
|
+
return f'{type(self).__name__}({len(self._forms)} forms)'
|
|
278
|
+
|
|
279
|
+
def mask(self, text: str) -> str:
|
|
280
|
+
pattern_ascii, pattern = self._pattern_ascii, self._pattern
|
|
281
|
+
if pattern is None or pattern_ascii is None:
|
|
282
|
+
return text
|
|
283
|
+
matches = pattern_ascii.finditer(text.lower()) if text.isascii() else pattern.finditer(text)
|
|
284
|
+
|
|
285
|
+
spans: list[list[int]] = []
|
|
286
|
+
for match in matches:
|
|
287
|
+
start, end = match.start(), match.start() + len(match[1])
|
|
288
|
+
if spans and start <= spans[-1][1]:
|
|
289
|
+
spans[-1][1] = max(spans[-1][1], end)
|
|
290
|
+
else:
|
|
291
|
+
spans.append([start, end])
|
|
292
|
+
if not spans:
|
|
293
|
+
return text
|
|
294
|
+
|
|
295
|
+
parts: list[str] = []
|
|
296
|
+
last = 0
|
|
297
|
+
for start, end in spans:
|
|
298
|
+
parts.append(text[last:start])
|
|
299
|
+
parts.append(MASKED_VALUE)
|
|
300
|
+
last = end
|
|
301
|
+
parts.append(text[last:])
|
|
302
|
+
return ''.join(parts)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from collections.abc import Callable
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
from .secrets import SecretsMasker
|
|
5
|
+
|
|
6
|
+
Processor = Callable[[Any, str, Any], Any]
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def mask_secrets_processor(masker: SecretsMasker) -> Processor:
|
|
10
|
+
"""A structlog processor that masks the rendered line.
|
|
11
|
+
|
|
12
|
+
Put it last, after the renderer: it then masks the whole line, keys and JSON-escaped forms included. It accepts
|
|
13
|
+
the output of a renderer, a `str` or `bytes`, and raises TypeError for anything else (an event dict): placed
|
|
14
|
+
before the renderer it could not mask the objects inside the event, and a silent partial masking is worse than a
|
|
15
|
+
failure at the first log call. The module does not import structlog: the processor is a plain function with the
|
|
16
|
+
`(logger, method_name, event_dict)` contract, the service brings its own structlog.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
def processor(logger: Any, method_name: str, event: Any) -> Any:
|
|
20
|
+
if isinstance(event, str):
|
|
21
|
+
return masker.mask(event)
|
|
22
|
+
if isinstance(event, bytes):
|
|
23
|
+
text = event.decode('utf-8', errors='surrogateescape')
|
|
24
|
+
return masker.mask(text).encode('utf-8', errors='surrogateescape')
|
|
25
|
+
raise TypeError(f'mask_secrets_processor must be the last processor, after the renderer; got {type(event).__name__}')
|
|
26
|
+
|
|
27
|
+
return processor
|
|
@@ -0,0 +1,379 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: logmask
|
|
3
|
+
Version: 0.1.3
|
|
4
|
+
Summary: JSON logging with masked secrets for Python services
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Author: Lido
|
|
8
|
+
Requires-Python: >=3.12,<4
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
14
|
+
Project-URL: Repository, https://github.com/lidofinance/logmask
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
# logmask
|
|
18
|
+
|
|
19
|
+
JSON logging for Python services, with secrets masked. Every log line is one JSON object. Every line goes through a masker that replaces every form of your secret values with `***`. This includes the messages of third-party libraries, warnings, and the traceback of an unhandled exception.
|
|
20
|
+
|
|
21
|
+
No runtime dependencies. Python 3.12 or newer.
|
|
22
|
+
|
|
23
|
+
## Install
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
poetry add git+https://github.com/lidofinance/logmask.git@v0.1.3
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Pin a tag, or a commit with `rev`, and commit `poetry.lock`: it records the exact commit. Poetry clones the repository itself; a build stage that installs with `pip` needs `git`.
|
|
30
|
+
|
|
31
|
+
## Quick start
|
|
32
|
+
|
|
33
|
+
```py
|
|
34
|
+
import logging
|
|
35
|
+
|
|
36
|
+
from logmask import masked_config, setup_logging
|
|
37
|
+
|
|
38
|
+
PUBLIC = {'RPC_TIMEOUT': 30, 'DRY_RUN': True}
|
|
39
|
+
PRIVATE = {'RPC_URL': 'https://rpc.example.com/v3/a1b2c3d4e5f6', 'API_TOKEN': 'tok_9f8e7d6c5b4a'}
|
|
40
|
+
|
|
41
|
+
setup_logging(PRIVATE)
|
|
42
|
+
|
|
43
|
+
logger = logging.getLogger(__name__)
|
|
44
|
+
logger.info({'msg': 'Config', 'value': masked_config(PUBLIC, PRIVATE)})
|
|
45
|
+
logger.info('Connecting to %s', PRIVATE['RPC_URL'])
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Output (one line per record, stderr):
|
|
49
|
+
|
|
50
|
+
```json
|
|
51
|
+
{"name": "app", "levelname": "INFO", "funcName": "main", "lineno": 12, "module": "main", "pathname": "/app/main.py", "timestamp": 1761600000, "msg": "Config", "value": {"RPC_TIMEOUT": 30, "DRY_RUN": true, "RPC_URL": "***", "API_TOKEN": "***"}}
|
|
52
|
+
{"name": "app", "levelname": "INFO", "funcName": "main", "lineno": 13, "module": "main", "pathname": "/app/main.py", "timestamp": 1761600000, "msg": "Connecting to ***"}
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
`setup_logging(PRIVATE)` does four things:
|
|
56
|
+
|
|
57
|
+
- configures the root logger: JSON lines to stderr, level INFO, old handlers removed
|
|
58
|
+
- masks every line with every form of the values of `PRIVATE`
|
|
59
|
+
- routes `warnings.warn()` through the logger, so a warning text is masked too
|
|
60
|
+
- installs `sys.excepthook`, `threading.excepthook` and `sys.unraisablehook`, so an unhandled exception becomes one CRITICAL line with a masked traceback
|
|
61
|
+
|
|
62
|
+
`masked_config(PUBLIC, PRIVATE)` builds a dict for the startup log. Public values stay as they are. Private values become `***`. A private list keeps its length. An unset private value (`None`, `''` or `b''`) becomes `''`, so you can see which variables are set; any other value, `0` and `False` included, is set and shows `***`. A key present in both dicts raises `ValueError`, and a private value of a type the masker cannot search raises `TypeError` (see "What is masked").
|
|
63
|
+
|
|
64
|
+
```py
|
|
65
|
+
masked_config({'PORT': 9000}, {'TOKEN': 'abc', 'RPC_URLS': ['a', 'b'], 'OPTIONAL_KEY': None})
|
|
66
|
+
# {'PORT': 9000, 'TOKEN': '***', 'RPC_URLS': ['***', '***'], 'OPTIONAL_KEY': ''}
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Every value of `PRIVATE` is treated as a secret, whatever it looks like (`None` and booleans are not values). Put a variable there only when its value must never appear in a log: a value such as `true` or `9000` in `PRIVATE` would be masked inside every line, field names included. A realistic set of secrets (about 20 variables, about 100 forms) costs about 50 microseconds per line; hundreds of secrets make every line noticeably slower.
|
|
70
|
+
|
|
71
|
+
## How to
|
|
72
|
+
|
|
73
|
+
### Log a message
|
|
74
|
+
|
|
75
|
+
A string message goes to the `msg` field. A dict message is merged into the line, so its keys become fields:
|
|
76
|
+
|
|
77
|
+
```py
|
|
78
|
+
logger.info('Report sent for slot %d', 1234567)
|
|
79
|
+
# {..., "msg": "Report sent for slot 1234567"}
|
|
80
|
+
|
|
81
|
+
logger.info({'msg': 'Report sent', 'slot': 1234567, 'root': b'\x0a\x0b'})
|
|
82
|
+
# {..., "msg": "Report sent", "slot": 1234567, "root": "0x0a0b"}
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
### Log an exception
|
|
86
|
+
|
|
87
|
+
```py
|
|
88
|
+
try:
|
|
89
|
+
client.get(url)
|
|
90
|
+
except ConnectionError:
|
|
91
|
+
logger.exception('Request failed')
|
|
92
|
+
# {..., "levelname": "ERROR", "msg": "Request failed", "exc_info": "Traceback (most recent call last):\n ...\nConnectionError: Max retries exceeded with url: ***"}
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
### Set the level and the stream
|
|
96
|
+
|
|
97
|
+
```py
|
|
98
|
+
setup_logging(PRIVATE, level=os.getenv('LOG_LEVEL', 'INFO'), stream=sys.stdout)
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
`level` takes a name in any case or an int. `stream` is stderr by default. Three outputs always go to stderr: the traceback of a KeyboardInterrupt, the report of a handler that cannot write, and the line written when the exception hook itself fails.
|
|
102
|
+
|
|
103
|
+
### Add secrets later
|
|
104
|
+
|
|
105
|
+
`setup_logging` returns the masker. Pass `{}` and call `add()` when a secret becomes known after logging is configured, for example a token fetched from a vault or a config module that logs at import time:
|
|
106
|
+
|
|
107
|
+
```py
|
|
108
|
+
masker = setup_logging({})
|
|
109
|
+
|
|
110
|
+
from app import config # logs at import time, before the config is known
|
|
111
|
+
|
|
112
|
+
masker.add(config.PRIVATE)
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Every line from that point on is masked with the new values too. The lines written before `add()` are not. A second `setup_logging` call in the same process adds its secrets to the same masker, so the secrets of the first call stay masked.
|
|
116
|
+
|
|
117
|
+
### Mask text by hand
|
|
118
|
+
|
|
119
|
+
```py
|
|
120
|
+
masker = setup_logging(PRIVATE)
|
|
121
|
+
masker.mask('connect https://rpc.example.com/v3/a1b2c3d4e5f6')
|
|
122
|
+
# 'connect ***'
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
### Use structlog
|
|
126
|
+
|
|
127
|
+
Call `setup_logging` as well: structlog does not route the standard-library loggers of `web3`, `urllib3` or `requests` anywhere, and without a configured root logger their warnings go to stderr unmasked.
|
|
128
|
+
|
|
129
|
+
```py
|
|
130
|
+
import structlog
|
|
131
|
+
|
|
132
|
+
from logmask import mask_secrets_processor, setup_logging
|
|
133
|
+
|
|
134
|
+
masker = setup_logging(PRIVATE)
|
|
135
|
+
structlog.configure(
|
|
136
|
+
processors=[
|
|
137
|
+
structlog.processors.add_log_level,
|
|
138
|
+
structlog.processors.TimeStamper(fmt='iso'),
|
|
139
|
+
structlog.processors.JSONRenderer(),
|
|
140
|
+
mask_secrets_processor(masker), # last: masks the rendered line
|
|
141
|
+
],
|
|
142
|
+
)
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
The processor must be the last one, after the renderer: it masks the rendered line, keys and JSON-escaped forms included. Placed earlier it receives the event dict and raises `TypeError` at the first event that reaches the processors, which with a filtering logger is the first event at or above the configured level, not the first call. Do not render tracebacks with `structlog.processors.dict_tracebacks` in its default form: it prints the local variables of every frame cut to 80 characters, and a cut secret is a part that no form matches. Use `format_exc_info`, or `ExceptionRenderer(ExceptionDictTransformer(show_locals=False))`. The package does not depend on structlog: your service installs its own version.
|
|
146
|
+
|
|
147
|
+
### Use the pieces one by one
|
|
148
|
+
|
|
149
|
+
`setup_logging` wires these classes; they can be used directly, and each of them requires the masker:
|
|
150
|
+
|
|
151
|
+
- `SecretsMasker(secrets)` - the masker; `mask(text)` and `add(secrets)`
|
|
152
|
+
- `JsonFormatter(masker)` - one JSON object per record, masked
|
|
153
|
+
- `SafeStreamHandler(stream, masker=masker)` - a `StreamHandler` that does not print the record when the write fails
|
|
154
|
+
- `install_excepthook(masker=masker)` - the three exception hooks; the masker covers the output that cannot go through the logger
|
|
155
|
+
|
|
156
|
+
## What is masked
|
|
157
|
+
|
|
158
|
+
The examples use this masker:
|
|
159
|
+
|
|
160
|
+
```py
|
|
161
|
+
masker = SecretsMasker(
|
|
162
|
+
{
|
|
163
|
+
'RPC_URL': 'https://rpc.example.com/v3/a1b2c3d4e5f6/',
|
|
164
|
+
'CL_URL': 'https://user:s3cretPassw0rd@beacon.example.com/',
|
|
165
|
+
'API_TOKEN': 'tok_9f8e7d6c5b4a\n', # read from a file, with the trailing newline
|
|
166
|
+
'RPC_LIST': ['https://one.example.com/key/abcdef123456', 'https://two.example.com/key/ghijkl789012'],
|
|
167
|
+
'KEYS': {'nested-secret-key': 'nested-secret-value'},
|
|
168
|
+
'PRIVATE_KEY': '0xABCDEF0123456789',
|
|
169
|
+
'RAW_KEY': b'\xde\xad\xbe\xef\x01\x02\x03\x04',
|
|
170
|
+
}
|
|
171
|
+
)
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
Each value, each item of a list, tuple, set or any other iterable, the keys and the values of a nested dict (only the top-level keys are the variable names, they stay), the fields of a dataclass, the value behind a `SecretStr`-like object (`get_secret_value()`), numbers and paths by their text:
|
|
175
|
+
|
|
176
|
+
```py
|
|
177
|
+
masker.mask('connect https://rpc.example.com/v3/a1b2c3d4e5f6/')
|
|
178
|
+
# 'connect ***'
|
|
179
|
+
masker.mask('rpc https://two.example.com/key/ghijkl789012 failed')
|
|
180
|
+
# 'rpc *** failed'
|
|
181
|
+
masker.mask('KEYS nested-secret-key nested-secret-value')
|
|
182
|
+
# 'KEYS *** ***'
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
A value of any other type (a plain object, a function, a module, a class) raises `TypeError` in `setup_logging`, `SecretsMasker`, `add()` and `masked_config`: the masker cannot promise to find every secret inside it, so it refuses the value instead of masking a part. Pass the data of such an object as a dict.
|
|
186
|
+
|
|
187
|
+
The values held by one string: a JSON document is searched as the container it encodes (a list of URLs, the JSON of a service account with its `private_key`), several URLs separated by commas, semicolons or white space are searched one by one, and every line of a multi-line value (a PEM key) is a secret on its own:
|
|
188
|
+
|
|
189
|
+
```py
|
|
190
|
+
masker = SecretsMasker({'RPC_LIST': 'https://one.example.com/key/abcdef123456,https://two.example.com/key/ghijkl789012'})
|
|
191
|
+
masker.mask('rpc https://one.example.com/key/abcdef123456 failed, path /key/abcdef123456')
|
|
192
|
+
# 'rpc *** failed, path ***'
|
|
193
|
+
masker = SecretsMasker(
|
|
194
|
+
{'ACCOUNT': '{"type": "service_account", "private_key": "-----BEGIN KEY-----\\nMIIEvQIBADANBg\\n-----END KEY-----"}'}
|
|
195
|
+
)
|
|
196
|
+
masker.mask('key MIIEvQIBADANBg of type service_account')
|
|
197
|
+
# 'key *** of *** ***'
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
The value without spaces and newlines around it, and without a trailing `/`:
|
|
201
|
+
|
|
202
|
+
```py
|
|
203
|
+
masker.mask('token tok_9f8e7d6c5b4a expired')
|
|
204
|
+
# 'token *** expired'
|
|
205
|
+
masker.mask('connect https://rpc.example.com/v3/a1b2c3d4e5f6')
|
|
206
|
+
# 'connect ***'
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
The parts of a URL on their own: the host with its credentials, the path, the path with the query, the query, each `name=value` pair, the user name, the password and `user:password`, percent-decoded as well, and the credentials as an HTTP Basic header. HTTP libraries often log only the path:
|
|
210
|
+
|
|
211
|
+
```py
|
|
212
|
+
masker.mask("HTTPSConnectionPool(host='rpc.example.com', port=443): Max retries exceeded with url: /v3/a1b2c3d4e5f6")
|
|
213
|
+
# "HTTPSConnectionPool(host='rpc.example.com', port=443): Max retries exceeded with url: ***"
|
|
214
|
+
masker.mask('GET https://user:s3cretPassw0rd@beacon.example.com/eth/v1/beacon/genesis')
|
|
215
|
+
# 'GET ***eth/v1/beacon/genesis'
|
|
216
|
+
masker.mask('auth user:s3cretPassw0rd rejected, header Basic dXNlcjpzM2NyZXRQYXNzdzByZA==')
|
|
217
|
+
# 'auth *** rejected, header Basic ***'
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
A single segment of the path, a single query value and the host without its credentials, when they can be a key: 16 characters or more, or 8 or more with a digit. A short word of a URL (`eth`, `v1`, a host like `el`) is masked only as part of the whole path or URL, otherwise it would vanish from every line:
|
|
221
|
+
|
|
222
|
+
```py
|
|
223
|
+
masker.mask('project a1b2c3d4e5f6 rate limited, host beacon.example.com')
|
|
224
|
+
# 'project *** rate limited, host ***'
|
|
225
|
+
masker.mask('GET /eth/v1/beacon/genesis from rpc.example.com')
|
|
226
|
+
# 'GET /eth/v1/beacon/genesis from rpc.example.com'
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
Hex values in any case, with and without `0x`, and as the bytes they encode. Bytes as text, as hex and as their `repr()`:
|
|
230
|
+
|
|
231
|
+
```py
|
|
232
|
+
masker.mask('key 0xabcdef0123456789 or ABCDEF0123456789 or 0XABCDEF0123456789')
|
|
233
|
+
# 'key *** or *** or ***'
|
|
234
|
+
masker.mask("raw 0xdeadbeef01020304 DEADBEEF01020304 b'\\xde\\xad\\xbe\\xef\\x01\\x02\\x03\\x04'")
|
|
235
|
+
# "raw *** *** b'***'"
|
|
236
|
+
masker.mask(repr(bytes.fromhex('ABCDEF0123456789')))
|
|
237
|
+
# "b'***'"
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
The value percent-encoded (with `quote` and with `quote_plus`), percent-decoded, base64-encoded, and as the hex or the `repr()` of its bytes, as an application writes it into a URL, a header or a bytes body:
|
|
241
|
+
|
|
242
|
+
```py
|
|
243
|
+
SecretsMasker({'T': 'tok/en+value=1'}).mask('k=tok%2Fen%2Bvalue%3D1 auth dG9rL2VuK3ZhbHVlPTE= body 0x746f6b2f656e2b76616c75653d31')
|
|
244
|
+
# 'k=*** auth *** body 0x***'
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
A value shown with `repr()`, inside `str(dict)`, in an exception text or inside a JSON document, up to two layers deep (for example the `repr()` of a JSON document). Quotes, backslashes and newlines in the secret are not a problem, and a secret made of JSON syntax (a quote, a brace, `true`) is masked inside the strings of the line, so the line stays valid JSON:
|
|
248
|
+
|
|
249
|
+
```py
|
|
250
|
+
password = 'pa"ss\\word\n123'
|
|
251
|
+
masker.add({'PASSWORD': password})
|
|
252
|
+
logger.info('login %r failed with %s', password, json.dumps({'password': password}))
|
|
253
|
+
# {..., "msg": "login '***' failed with {\"password\": \"***\"}"}
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
The case of the text does not matter:
|
|
257
|
+
|
|
258
|
+
```py
|
|
259
|
+
masker.mask('TOK_9F8E7D6C5B4A')
|
|
260
|
+
# '***'
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
Numbers and other non-string values, by their text:
|
|
264
|
+
|
|
265
|
+
```py
|
|
266
|
+
SecretsMasker({'PIN': 12345678}).mask('pin 12345678')
|
|
267
|
+
# 'pin ***'
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
Two secrets that overlap in the text are masked as one:
|
|
271
|
+
|
|
272
|
+
```py
|
|
273
|
+
SecretsMasker({'A': 'abcdefgh', 'B': 'efghijklmnop'}).mask('abcdefghijklmnop')
|
|
274
|
+
# '***'
|
|
275
|
+
```
|
|
276
|
+
|
|
277
|
+
## What is not masked
|
|
278
|
+
|
|
279
|
+
A value that is not in your secrets. The masker replaces known values, it does not search for text that looks like a secret:
|
|
280
|
+
|
|
281
|
+
```py
|
|
282
|
+
masker.mask('other secret zzz-not-configured')
|
|
283
|
+
# 'other secret zzz-not-configured'
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
A form of the value the masker does not know: a part of the value, a hash of it, more than two layers of escaping:
|
|
287
|
+
|
|
288
|
+
```py
|
|
289
|
+
masker.mask('prefix a1b2c3')
|
|
290
|
+
# 'prefix a1b2c3'
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
A short word of a URL on its own (see above).
|
|
294
|
+
|
|
295
|
+
A password re-encoded by an HTTP library with its own set of safe characters. The masker knows the password raw, percent-decoded and in the three encodings of `urllib.parse` (`quote` with and without the default safe characters, `quote_plus`). yarl, used by aiohttp, keeps `+ ! & =` raw and encodes the rest, so `p%40ss+x!y` is a form the masker does not know, and only the user name and the host of such a URL are masked. A per-character pattern for the credentials (each character raw or percent-encoded) is planned; until then, prefer a password without characters that need encoding.
|
|
296
|
+
|
|
297
|
+
Text that does not pass through the handler installed by `setup_logging`:
|
|
298
|
+
|
|
299
|
+
- `print`, a library that writes to the stream directly, and the message of `sys.exit('...')`, which the interpreter prints without calling the exception hook
|
|
300
|
+
- a logger with its own handler that does not propagate to the root logger (a logger without any handler that does not propagate goes to `logging.lastResort`, which `setup_logging` replaces by a masked handler)
|
|
301
|
+
- a handler added to the root logger after `setup_logging` (a `FileHandler`, the handler of an error tracker), and a `logging.config.dictConfig` call after it: they format the record themselves
|
|
302
|
+
- a child process of `multiprocessing`: with the `fork` start method, an exception in the child is printed by `multiprocessing` itself, without the hooks; with `spawn`, the child starts without `setup_logging` unless its module calls it
|
|
303
|
+
|
|
304
|
+
## When logging itself fails
|
|
305
|
+
|
|
306
|
+
A record that cannot be formatted (wrong `%` arguments, a value `json` cannot serialize, a dict message nested deeper than the recursion limit) is still written as one masked JSON line, with the problem in the `error` field. Every line the library writes on its own behalf, instead of the record, starts with `[logmask]`:
|
|
307
|
+
|
|
308
|
+
```py
|
|
309
|
+
logger.info('rpc %d %s', 'https://rpc.example.com/v3/a1b2c3d4e5f6', 'x')
|
|
310
|
+
# {..., "msg": "rpc %d %s", "error": "[logmask] Log record could not be formatted: TypeError('%d format: a real number is required, not str')", "args": "('***', 'x')"}
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
The standard library would print this instead, to stderr and without masking:
|
|
314
|
+
|
|
315
|
+
```text
|
|
316
|
+
--- Logging error ---
|
|
317
|
+
Traceback (most recent call last):
|
|
318
|
+
...
|
|
319
|
+
TypeError: %d format: a real number is required, not str
|
|
320
|
+
Message: 'rpc %d %s'
|
|
321
|
+
Arguments: ('https://rpc.example.com/v3/a1b2c3d4e5f6', 'x')
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
A record that cannot be written (the stream is closed, the pipe is broken) is reported by `SafeStreamHandler` without the record, where the standard handler would print the same `Message:` and `Arguments:` lines as above:
|
|
325
|
+
|
|
326
|
+
```text
|
|
327
|
+
--- [logmask] Logging error: BrokenPipeError(32, 'Broken pipe') (record from app, /app/main.py:31) ---
|
|
328
|
+
```
|
|
329
|
+
|
|
330
|
+
An unhandled exception goes through the logger as a CRITICAL line:
|
|
331
|
+
|
|
332
|
+
```py
|
|
333
|
+
raise ConnectionError(f'Max retries exceeded with url: {RPC_URL}')
|
|
334
|
+
# {"name": "unhandled", "levelname": "CRITICAL", ..., "msg": "Unhandled exception. Shut down.", "error": "Traceback (most recent call last):\n ...\nConnectionError: Max retries exceeded with url: ***\n"}
|
|
335
|
+
```
|
|
336
|
+
|
|
337
|
+
An exception in another thread gives the same line with `"msg": "Unhandled exception in a thread."` and a `thread` field; the process goes on, as with the default hook. An exception in `__del__` or in a callback (`sys.unraisablehook`) gives `"msg": "Unraisable exception."`.
|
|
338
|
+
|
|
339
|
+
Three cases cannot go through the logger and are written to stderr masked: a `KeyboardInterrupt` (Ctrl-C is not an error, so there is no CRITICAL line, the traceback is printed as plain text), an exception when no handler is configured (one JSON line written by the hook itself), and a failure of the hook (one line with the exception name).
|
|
340
|
+
|
|
341
|
+
## Log format
|
|
342
|
+
|
|
343
|
+
Every line has these fields, in this order:
|
|
344
|
+
|
|
345
|
+
```json
|
|
346
|
+
{"name": "app", "levelname": "INFO", "funcName": "main", "lineno": 29, "module": "main", "pathname": "/app/main.py", "timestamp": 1761600000, "msg": "Config", "value": {"RPC_URL": "***"}}
|
|
347
|
+
```
|
|
348
|
+
|
|
349
|
+
- `name`, `levelname`, `funcName`, `lineno`, `module`, `pathname` - from the log record
|
|
350
|
+
- `timestamp` - the record time in whole seconds
|
|
351
|
+
- `msg` - the string message; a dict message adds its keys after `timestamp` instead. A dict key with the name of a record field (for example `name`) is written as `msg_name`
|
|
352
|
+
- `exc_info` - the traceback, for `logger.exception()` and `exc_info=True`
|
|
353
|
+
- bytes are written as `0x`-hex, dataclasses as dicts, sets as lists, other unknown values as text
|
|
354
|
+
|
|
355
|
+
## Development
|
|
356
|
+
|
|
357
|
+
```bash
|
|
358
|
+
poetry install
|
|
359
|
+
poetry run ruff format --check . && poetry run ruff check . && poetry run pyright logmask tests
|
|
360
|
+
poetry run pytest
|
|
361
|
+
```
|
|
362
|
+
|
|
363
|
+
Mutation tests check that the suite notices a weaker masker. They run in CI on every push and pull request; locally:
|
|
364
|
+
|
|
365
|
+
```bash
|
|
366
|
+
poetry run pip install "mutmut==3.8.1"
|
|
367
|
+
poetry run mutmut run
|
|
368
|
+
poetry run mutmut results # the survivors
|
|
369
|
+
poetry run mutmut show <mutant> # the diff of one survivor
|
|
370
|
+
```
|
|
371
|
+
|
|
372
|
+
About 50 mutants survive by design (the spelling of `utf-8`, the wording of messages, the references kept by the cycle guard); CI fails above that number.
|
|
373
|
+
|
|
374
|
+
## Release
|
|
375
|
+
|
|
376
|
+
1. Set the version in `pyproject.toml` and move the `Unreleased` section of `CHANGELOG.md` under the version and the date.
|
|
377
|
+
2. Merge to `main` and create the tag `vX.Y.Z`.
|
|
378
|
+
3. Consumers update the tag in their `pyproject.toml` and run `poetry lock`.
|
|
379
|
+
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
logmask/__init__.py,sha256=UlxYUfyJKYTg1xsvyht2UDGLSxoq4kuP_a88emAWgPA,495
|
|
2
|
+
logmask/config.py,sha256=yT6bMassP-nVaIoPv4LsBlUME1dOdr6LsDLCCvX57EM,1289
|
|
3
|
+
logmask/configure.py,sha256=HBVX3m6lNf3_8mofkfDamsCE5d9llbMTcMj4KhN0AHA,2444
|
|
4
|
+
logmask/excepthook.py,sha256=bqbsJcZbXtBwbcp0W2hWm1FUEZne0y41kCUmgbNDmtk,4831
|
|
5
|
+
logmask/formatter.py,sha256=RoCY3DNmD232oen27bLjtZ2jCKkIx4YlBAPsrad14xQ,4672
|
|
6
|
+
logmask/handler.py,sha256=VWXgv8vcIGvLzB1dRhO99fmRXeSGDSz_lzwcibni9WE,1163
|
|
7
|
+
logmask/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
logmask/secrets.py,sha256=aStieUQ-U6H-xkmNdCfxAhUaK4EaRFABwSfaQzpSIr8,13293
|
|
9
|
+
logmask/structlog_processor.py,sha256=2xhxFbqaBUYa-SQPEtiFZgbBt6KrcLiG30735U190Xc,1298
|
|
10
|
+
logmask-0.1.3.dist-info/METADATA,sha256=DOW4BgTTH6w-3SD3my9hqCr4UY0dBcc2zKs9k7czOjM,18594
|
|
11
|
+
logmask-0.1.3.dist-info/WHEEL,sha256=L-WvLdSBvJWHXv5zrQxDvNRnIgoBzLUpcbuiw19KCuQ,88
|
|
12
|
+
logmask-0.1.3.dist-info/licenses/LICENSE,sha256=LpEi6KieJAlefSgmC1tWdhIUzPCoAis2DEZRzQbD6e0,1061
|
|
13
|
+
logmask-0.1.3.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Lido
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|