logdruid 0.1.0.dev1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Frank Herbrand
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,97 @@
1
+ Metadata-Version: 2.4
2
+ Name: logdruid
3
+ Version: 0.1.0.dev1
4
+ Summary: A drop-in logging setup layer for multi-process applications, built on loguru.
5
+ License-Expression: MIT
6
+ License-File: LICENSE
7
+ Author: Frank Herbrand
8
+ Author-email: f.herbrand@hzdr.de
9
+ Requires-Python: >=3.11,<4.0
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Programming Language :: Python :: 3.11
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Classifier: Programming Language :: Python :: 3.13
14
+ Classifier: Programming Language :: Python :: 3.14
15
+ Classifier: Programming Language :: Python :: 3.15
16
+ Requires-Dist: loguru (>=0.7.3)
17
+ Description-Content-Type: text/markdown
18
+
19
+ # logdruid
20
+
21
+ [![PyPI version](https://img.shields.io/pypi/v/logdruid.svg)](https://pypi.org/project/logdruid/)
22
+ [![Tests](https://github.com/fherb2/logdruid/actions/workflows/tests.yml/badge.svg)](https://github.com/fherb2/logdruid/actions/workflows/tests.yml)
23
+
24
+ A drop-in logging setup layer for multi-process applications, built on
25
+ [loguru](https://loguru.readthedocs.io/).
26
+
27
+ logdruid does not replace loguru's API — it uses it unchanged — and adds
28
+ what a multi-process application needs on top: a single, time-ordered
29
+ output across process boundaries, without a lock shared between
30
+ processes, and the ability to take over messages from libraries that
31
+ report through the standard `logging` module.
32
+
33
+ ## Installation
34
+
35
+ ```bash
36
+ pip install logdruid
37
+ ```
38
+
39
+ Requires Python 3.11 or later.
40
+
41
+ ## Usage
42
+
43
+ Main process, once, before any child process is created:
44
+
45
+ ```python
46
+ import logdruid
47
+ from logdruid import logger
48
+
49
+ logdruid.setup(config_path, cli_level=args.log_level)
50
+ logger.success("Logging active")
51
+ ```
52
+
53
+ Per child process, in the main process:
54
+
55
+ ```python
56
+ link = logdruid.link("raw_writer")
57
+ process = ctx.Process(target=raw_writer_prc, args=(link, ...),
58
+ name="raw_writer")
59
+ process.start()
60
+ link.release()
61
+ ```
62
+
63
+ First thing in the child process (after ignoring `SIGINT`):
64
+
65
+ ```python
66
+ logdruid.attach(link)
67
+ ```
68
+
69
+ At the end of the main process, after the children have ended:
70
+
71
+ ```python
72
+ logdruid.shutdown()
73
+ ```
74
+
75
+ Application modules report through `from logdruid import logger` — loguru's
76
+ own global logger, unchanged. Libraries that report through the standard
77
+ `logging` module need no changes of their own; logdruid takes over their
78
+ messages and applies the level translations configured for them.
79
+
80
+ ## Configuration
81
+
82
+ Everything logdruid-specific lives in a `logdruid.toml` file next to the
83
+ application's entry point. A fully commented example, including how to
84
+ assign levels per process and per logger and how to translate a library's
85
+ levels into the application's own scheme, is in
86
+ [`examples/logdruid.toml`](examples/logdruid.toml).
87
+
88
+ ## Documentation
89
+
90
+ The full set of project-wide rules and design rationale is in
91
+ [`running_implementation_doc/implementierungsdoku.md`](running_implementation_doc/implementierungsdoku.md)
92
+ (in German).
93
+
94
+ ## License
95
+
96
+ MIT, see [`LICENSE`](LICENSE).
97
+
@@ -0,0 +1,78 @@
1
+ # logdruid
2
+
3
+ [![PyPI version](https://img.shields.io/pypi/v/logdruid.svg)](https://pypi.org/project/logdruid/)
4
+ [![Tests](https://github.com/fherb2/logdruid/actions/workflows/tests.yml/badge.svg)](https://github.com/fherb2/logdruid/actions/workflows/tests.yml)
5
+
6
+ A drop-in logging setup layer for multi-process applications, built on
7
+ [loguru](https://loguru.readthedocs.io/).
8
+
9
+ logdruid does not replace loguru's API — it uses it unchanged — and adds
10
+ what a multi-process application needs on top: a single, time-ordered
11
+ output across process boundaries, without a lock shared between
12
+ processes, and the ability to take over messages from libraries that
13
+ report through the standard `logging` module.
14
+
15
+ ## Installation
16
+
17
+ ```bash
18
+ pip install logdruid
19
+ ```
20
+
21
+ Requires Python 3.11 or later.
22
+
23
+ ## Usage
24
+
25
+ Main process, once, before any child process is created:
26
+
27
+ ```python
28
+ import logdruid
29
+ from logdruid import logger
30
+
31
+ logdruid.setup(config_path, cli_level=args.log_level)
32
+ logger.success("Logging active")
33
+ ```
34
+
35
+ Per child process, in the main process:
36
+
37
+ ```python
38
+ link = logdruid.link("raw_writer")
39
+ process = ctx.Process(target=raw_writer_prc, args=(link, ...),
40
+ name="raw_writer")
41
+ process.start()
42
+ link.release()
43
+ ```
44
+
45
+ First thing in the child process (after ignoring `SIGINT`):
46
+
47
+ ```python
48
+ logdruid.attach(link)
49
+ ```
50
+
51
+ At the end of the main process, after the children have ended:
52
+
53
+ ```python
54
+ logdruid.shutdown()
55
+ ```
56
+
57
+ Application modules report through `from logdruid import logger` — loguru's
58
+ own global logger, unchanged. Libraries that report through the standard
59
+ `logging` module need no changes of their own; logdruid takes over their
60
+ messages and applies the level translations configured for them.
61
+
62
+ ## Configuration
63
+
64
+ Everything logdruid-specific lives in a `logdruid.toml` file next to the
65
+ application's entry point. A fully commented example, including how to
66
+ assign levels per process and per logger and how to translate a library's
67
+ levels into the application's own scheme, is in
68
+ [`examples/logdruid.toml`](examples/logdruid.toml).
69
+
70
+ ## Documentation
71
+
72
+ The full set of project-wide rules and design rationale is in
73
+ [`running_implementation_doc/implementierungsdoku.md`](running_implementation_doc/implementierungsdoku.md)
74
+ (in German).
75
+
76
+ ## License
77
+
78
+ MIT, see [`LICENSE`](LICENSE).
@@ -0,0 +1,30 @@
1
+ [build-system]
2
+ requires = ["poetry-core"]
3
+ build-backend = "poetry.core.masonry.api"
4
+
5
+ [project]
6
+ name = "logdruid"
7
+ version = "0.1.0.dev1"
8
+ description = "A drop-in logging setup layer for multi-process applications, built on loguru."
9
+ readme = "README.md"
10
+ requires-python = ">=3.11,<4.0"
11
+ license = "MIT"
12
+ license-files = ["LICENSE"]
13
+ authors = [
14
+ { name = "Frank Herbrand", email = "f.herbrand@hzdr.de" },
15
+ ]
16
+ dependencies = [
17
+ "loguru>=0.7.3",
18
+ ]
19
+
20
+ [tool.poetry]
21
+ packages = [{ include = "logdruid", from = "src" }]
22
+
23
+ [tool.poetry.group.dev.dependencies]
24
+ tox = "^4.0"
25
+ tox-gh = "^1.0"
26
+
27
+ [tool.poetry.group.docs.dependencies]
28
+ mkdocs = "^1.6"
29
+ mkdocs-material = "^9.5"
30
+ mkdocstrings = { version = ">=0.27", extras = ["python"] }
@@ -0,0 +1,86 @@
1
+ """Logging setup for multi-process applications: a setup layer around loguru.
2
+
3
+ Reporting uses loguru unchanged: `logger` is loguru's global logger.
4
+ This module only sets it up, and adds what loguru does not offer:
5
+
6
+ - One output in the main process. Console and file are created there
7
+ only; every child process sends its messages through a pipe of its
8
+ own. Unlike loguru's ``enqueue=True`` no lock is shared between
9
+ processes, so a child that is killed hard blocks nobody but itself.
10
+ - Ordering by time. The main process holds messages back for a short
11
+ window and writes them sorted by their time stamp, its own included.
12
+ - Taking over the standard ``logging`` module, with the logger name as
13
+ the name of the message and with levels translated per logger.
14
+ - Levels per process and per name, configured in ``logdruid.toml`` in
15
+ four layers: module defaults < API < file < command line.
16
+
17
+ Usage
18
+ -----
19
+ Main process, once, before any child process is created:
20
+
21
+ >>> import logdruid
22
+ >>> from logdruid import logger
23
+ >>> logdruid.setup(config_path, cli_level=args.log_level)
24
+ >>> logger.success("Logging active")
25
+
26
+ Per child process, in the main process:
27
+
28
+ >>> link = logdruid.link("raw_writer")
29
+ >>> process = ctx.Process(target=raw_writer_prc, args=(link, ...),
30
+ ... name="raw_writer")
31
+ >>> process.start()
32
+ >>> link.release()
33
+
34
+ First thing in the child process (after ignoring SIGINT):
35
+
36
+ >>> logdruid.attach(link)
37
+
38
+ At the end of the main process, after the children have ended:
39
+
40
+ >>> logdruid.shutdown()
41
+
42
+ Configuration file (all keys optional)::
43
+
44
+ std_level = "INFO" # level of everything without its own
45
+ sort_window_ms = 100 # 0 switches the ordering off
46
+
47
+ [output.console] # parameters of loguru's add(),
48
+ enabled = true # passed through; 'enabled' and
49
+ format = "..." # 'path' are logdruid's own
50
+ diagnose = false
51
+
52
+ [output.file]
53
+ enabled = false
54
+ path = "app.log"
55
+ rotation = "10 MB"
56
+ retention = 5
57
+
58
+ [processes.raw_writer]
59
+ level = "DEBUG"
60
+
61
+ [loggers.ognbligg] # applies to ognbligg.* as well
62
+ level = "INFO"
63
+ translate = { DEBUG = "TRACE", INFO = false } # false drops
64
+
65
+ Notes on further development
66
+ ----------------------------
67
+ - An application started through a packaged entry point has no
68
+ ``logdruid.toml`` next to ``__main__``; it passes the path instead.
69
+ - Exceptions travel through the pipe as text formatted in the child;
70
+ ``backtrace`` and ``diagnose`` therefore act where a message arises.
71
+ """
72
+
73
+
74
+ from .logdruid import (
75
+ CONFIG_FILE_NAME,
76
+ LogLink,
77
+ Settings,
78
+ attach,
79
+ link,
80
+ logger,
81
+ setup,
82
+ shutdown,
83
+ )
84
+
85
+ __all__ = ["logger", "setup", "link", "attach", "shutdown", "LogLink",
86
+ "Settings", "CONFIG_FILE_NAME"]
@@ -0,0 +1,684 @@
1
+ """Implementation of the logdruid package.
2
+
3
+ Public interface and usage: see the package docstring in
4
+ ``logdruid/__init__.py``.
5
+ """
6
+
7
+ import atexit
8
+ import copy
9
+ import heapq
10
+ import inspect
11
+ import itertools
12
+ import logging
13
+ import multiprocessing as mp
14
+ import sys
15
+ import threading
16
+ import time
17
+ import tomllib
18
+ from dataclasses import dataclass
19
+ from multiprocessing.connection import wait
20
+ from pathlib import Path
21
+ from typing import Any, Dict, List, Optional
22
+
23
+ from loguru import logger
24
+
25
+ # --- Module parameters ------------------------------------------------
26
+
27
+ # Name of the configuration file looked for next to the entry point.
28
+ CONFIG_FILE_NAME = "logdruid.toml"
29
+
30
+ # Lowest layer: what applies when neither API, file nor command line
31
+ # say anything. No translation, one level for everything.
32
+ MODULE_DEFAULTS: Dict[str, Any] = {
33
+ "std_level": "INFO",
34
+ "sort_window_ms": 100,
35
+ "output": {
36
+ "console": {"enabled": True, "diagnose": False},
37
+ "file": {"enabled": False, "diagnose": False},
38
+ },
39
+ }
40
+
41
+ # Keys of an output table that logdruid handles itself.
42
+ OWN_OUTPUT_KEYS = {"enabled", "path"}
43
+
44
+ # Parameters of loguru's add() that logdruid sets itself or forbids:
45
+ # level and filter come from the configuration, the sink from the table,
46
+ # and enqueue would bring back the shared lock.
47
+ RESERVED_OUTPUT_KEYS = {"sink", "level", "filter", "enqueue", "context"}
48
+
49
+ # Parameters loguru accepts for file sinks only (loguru API, add()).
50
+ FILE_SINK_KEYS = {"rotation", "retention", "compression", "delay", "watch",
51
+ "mode", "buffering", "encoding"}
52
+
53
+ # Format of the handlers that collect or send messages. loguru appends a
54
+ # newline and the formatted exception to every string format by itself,
55
+ # so the text carries the traceback, which cannot be sent as an object.
56
+ COLLECT_FORMAT = "{message}"
57
+
58
+ # Upper bound on reading the pipes to their end in shutdown(). They end
59
+ # by themselves once the children have ended; this only keeps a forgotten
60
+ # release() from hanging the exit.
61
+ SHUTDOWN_TIMEOUT_S = 5.0
62
+
63
+ # Longest wait for a pipe to become readable when no message is due. It
64
+ # bounds how late a newly linked pipe is watched.
65
+ COLLECT_POLL_S = 0.05
66
+
67
+ # --- Module state -----------------------------------------------------
68
+
69
+ # Resolved configuration of this process; set by setup() or attach().
70
+ _settings: Optional["Settings"] = None
71
+
72
+ # Threshold per (process, name), filled on first use.
73
+ _threshold_cache: Dict[tuple, int] = {}
74
+
75
+ # Main process only: the collector and the output logger it writes to.
76
+ _collector: Optional["_Collector"] = None
77
+ _output = None
78
+
79
+
80
+ @dataclass
81
+ class Settings:
82
+ """Resolved configuration, small enough to go to a child process.
83
+
84
+ Attributes
85
+ ----------
86
+ layers : list of dict
87
+ One entry per layer, highest priority first. Each holds
88
+ ``std_level`` (level number or None), ``processes`` (process
89
+ name -> level number) and ``loggers`` (name -> dict with an
90
+ optional ``level`` number and an optional ``translate`` mapping
91
+ of level names to level names or False).
92
+ sort_window_s : float
93
+ Ordering window in seconds; 0 switches the ordering off.
94
+ outputs : dict
95
+ Enabled outputs ('console', 'file') with the keyword arguments
96
+ for loguru's add(), the file path under 'path'.
97
+ backtrace : bool
98
+ Passed to the handlers that format exceptions where they arise.
99
+ diagnose : bool
100
+ Same, for the display of variable values in tracebacks.
101
+ """
102
+
103
+ layers: List[Dict[str, Any]]
104
+ sort_window_s: float
105
+ outputs: Dict[str, Dict[str, Any]]
106
+ backtrace: bool
107
+ diagnose: bool
108
+
109
+
110
+ @dataclass
111
+ class LogLink:
112
+ """What a child process needs to log into the main process.
113
+
114
+ Created by link() in the main process and handed to the child as an
115
+ argument; the child passes it to attach().
116
+
117
+ Attributes
118
+ ----------
119
+ process_name : str
120
+ Name the child process carries in every message.
121
+ conn : multiprocessing.connection.Connection
122
+ Write end of the child's own pipe.
123
+ settings : Settings
124
+ Resolved configuration of the main process.
125
+ """
126
+
127
+ process_name: str
128
+ conn: Any
129
+ settings: Settings
130
+
131
+ def release(self) -> None:
132
+ """Drop the main process's copy of the write end.
133
+
134
+ Call right after the child process has started. Without it the
135
+ pipe never reaches its end, and shutdown() waits for nothing.
136
+ """
137
+ self.conn.close()
138
+
139
+
140
+ # --- Public functions -------------------------------------------------
141
+
142
+ def setup(config_path: Optional[Path] = None, *,
143
+ defaults: Optional[Dict[str, Any]] = None,
144
+ cli_level: Optional[str] = None,
145
+ cli_log_file: Optional[str] = None,
146
+ process_name: str = "main") -> None:
147
+ """Set up logging in the main process.
148
+
149
+ Call once, before any child process is created. Problems in the
150
+ configuration are reported as warnings once the output exists; they
151
+ never stop the application.
152
+
153
+ Parameters
154
+ ----------
155
+ config_path : Path, optional
156
+ Configuration file. Without it, ``logdruid.toml`` is looked for
157
+ next to the entry point of the application.
158
+ defaults : dict, optional
159
+ Settings of the application, laid over the module defaults and
160
+ under the file; same layout as the file.
161
+ cli_level : str, optional
162
+ Level of everything, from the command line. Overrides every
163
+ level of file and API; translations stay.
164
+ cli_log_file : str, optional
165
+ Log file from the command line. Takes effect only when the file
166
+ output is enabled.
167
+ process_name : str
168
+ Name this process carries in its messages.
169
+ """
170
+ global _settings, _collector, _output
171
+
172
+ mp.current_process().name = process_name
173
+ warnings: List[str] = []
174
+ file_layer = _read_config_file(config_path, warnings)
175
+ cli_layer: Dict[str, Any] = {}
176
+ if cli_level is not None:
177
+ cli_layer["std_level"] = cli_level
178
+ if cli_log_file is not None:
179
+ cli_layer["output"] = {"file": {"path": cli_log_file}}
180
+
181
+ raw_layers = [cli_layer, file_layer, defaults or {}, MODULE_DEFAULTS]
182
+ _settings = _resolve(raw_layers, warnings)
183
+ _threshold_cache.clear()
184
+
185
+ logger.remove()
186
+ _output = copy.deepcopy(logger)
187
+ _add_outputs(_output, _settings, warnings)
188
+
189
+ _collector = _Collector(_output, _settings.sort_window_s)
190
+ _collector.start()
191
+ logger.add(_collector.collect, level=_min_threshold(process_name),
192
+ filter=_level_filter, format=COLLECT_FORMAT,
193
+ backtrace=_settings.backtrace, diagnose=_settings.diagnose)
194
+ _take_over_std_logging(process_name)
195
+ atexit.register(shutdown)
196
+
197
+ for text in warnings:
198
+ logger.warning(text)
199
+
200
+
201
+ def link(process_name: str) -> LogLink:
202
+ """Prepare logging for one child process; main process only.
203
+
204
+ Parameters
205
+ ----------
206
+ process_name : str
207
+ Name the child process will carry.
208
+
209
+ Returns
210
+ -------
211
+ LogLink
212
+ To be handed to the child process as an argument.
213
+ """
214
+ if _collector is None:
215
+ raise RuntimeError("logdruid.setup() has to come first")
216
+ read_end, write_end = mp.get_context("spawn").Pipe(duplex=False)
217
+ _collector.add_pipe(read_end, process_name)
218
+ return LogLink(process_name, write_end, _settings)
219
+
220
+
221
+ def attach(log_link: LogLink) -> None:
222
+ """Set up logging in a child process.
223
+
224
+ Call first in the process function, after ignoring SIGINT. Replaces
225
+ loguru's default output by a handler that writes into the pipe of
226
+ this process, and takes over the standard ``logging`` module.
227
+
228
+ Parameters
229
+ ----------
230
+ log_link : LogLink
231
+ Handed over by the main process.
232
+ """
233
+ global _settings
234
+
235
+ mp.current_process().name = log_link.process_name
236
+ _settings = log_link.settings
237
+ _threshold_cache.clear()
238
+ logger.remove()
239
+ logger.add(_PipeSink(log_link.conn),
240
+ level=_min_threshold(log_link.process_name),
241
+ filter=_level_filter, format=COLLECT_FORMAT,
242
+ backtrace=_settings.backtrace, diagnose=_settings.diagnose)
243
+ _take_over_std_logging(log_link.process_name)
244
+
245
+
246
+ def shutdown() -> None:
247
+ """Write out everything and close the output; main process only.
248
+
249
+ Call at the end, after the child processes have ended. Reads every
250
+ pipe to its end, writes what is still held back, and from then on
251
+ writes the main process's messages directly, unsorted. Safe to call
252
+ more than once.
253
+ """
254
+ if _collector is None or not _collector.is_alive():
255
+ return
256
+ _collector.stop(SHUTDOWN_TIMEOUT_S)
257
+
258
+
259
+ # --- Configuration ----------------------------------------------------
260
+
261
+ def _read_config_file(config_path: Optional[Path],
262
+ warnings: List[str]) -> Dict[str, Any]:
263
+ """Return the file layer, or an empty one with a warning."""
264
+ if config_path is None:
265
+ main_file = getattr(sys.modules.get("__main__"), "__file__", None)
266
+ if main_file is None:
267
+ warnings.append(f"No entry point file, so no {CONFIG_FILE_NAME} "
268
+ f"is looked for; using API and defaults only")
269
+ return {}
270
+ config_path = Path(main_file).resolve().parent / CONFIG_FILE_NAME
271
+ try:
272
+ with open(config_path, "rb") as file:
273
+ return tomllib.load(file)
274
+ except FileNotFoundError:
275
+ warnings.append(f"{config_path} not found; using API and defaults "
276
+ f"only")
277
+ except (OSError, tomllib.TOMLDecodeError) as exc:
278
+ warnings.append(f"{config_path} cannot be read ({exc}); using API "
279
+ f"and defaults only")
280
+ return {}
281
+
282
+
283
+ def _resolve(raw_layers: List[Dict[str, Any]],
284
+ warnings: List[str]) -> Settings:
285
+ """Turn the raw layers, highest first, into Settings."""
286
+ layers = [_resolve_levels(raw, warnings) for raw in raw_layers]
287
+
288
+ window_ms = next((raw["sort_window_ms"] for raw in raw_layers
289
+ if "sort_window_ms" in raw), 0)
290
+ if not isinstance(window_ms, (int, float)) or window_ms < 0:
291
+ warnings.append(f"sort_window_ms must be a number >= 0, not "
292
+ f"{window_ms!r}; ordering switched off")
293
+ window_ms = 0
294
+
295
+ outputs: Dict[str, Dict[str, Any]] = {}
296
+ for kind in ("console", "file"):
297
+ merged: Dict[str, Any] = {}
298
+ for raw in reversed(raw_layers):
299
+ merged.update(raw.get("output", {}).get(kind, {}))
300
+ if merged.pop("enabled", False):
301
+ outputs[kind] = _checked_output(kind, merged, warnings)
302
+ if "file" in outputs and not outputs["file"].get("path"):
303
+ warnings.append("File output enabled without a path; switched off")
304
+ del outputs["file"]
305
+ if not outputs:
306
+ warnings.append("No output enabled; writing to the console")
307
+ outputs["console"] = {}
308
+
309
+ backtrace = any(params.pop("backtrace", True)
310
+ for params in outputs.values())
311
+ diagnose = any(params.pop("diagnose", False)
312
+ for params in outputs.values())
313
+ return Settings(layers, window_ms / 1000, outputs, backtrace, diagnose)
314
+
315
+
316
+ def _resolve_levels(raw: Dict[str, Any],
317
+ warnings: List[str]) -> Dict[str, Any]:
318
+ """Pick the levels and translations out of one raw layer."""
319
+ known = {"std_level", "sort_window_ms", "output", "processes", "loggers"}
320
+ for key in raw.keys() - known:
321
+ warnings.append(f"Unknown logging setting '{key}' ignored")
322
+
323
+ layer: Dict[str, Any] = {
324
+ "std_level": _level_no(raw.get("std_level"), "std_level", warnings),
325
+ "processes": {},
326
+ "loggers": {},
327
+ }
328
+ for name, entry in raw.get("processes", {}).items():
329
+ for key in entry.keys() - {"level"}:
330
+ warnings.append(f"Unknown key '{key}' for process '{name}' "
331
+ f"ignored")
332
+ level = _level_no(entry.get("level"), f"process '{name}'", warnings)
333
+ if level is not None:
334
+ layer["processes"][name] = level
335
+ for name, entry in raw.get("loggers", {}).items():
336
+ for key in entry.keys() - {"level", "translate"}:
337
+ warnings.append(f"Unknown key '{key}' for logger '{name}' "
338
+ f"ignored")
339
+ resolved: Dict[str, Any] = {}
340
+ level = _level_no(entry.get("level"), f"logger '{name}'", warnings)
341
+ if level is not None:
342
+ resolved["level"] = level
343
+ if "translate" in entry:
344
+ resolved["translate"] = _checked_translation(
345
+ name, entry["translate"], warnings)
346
+ if resolved:
347
+ layer["loggers"][name] = resolved
348
+ return layer
349
+
350
+
351
+ def _level_no(level: Any, what: str, warnings: List[str]) -> Optional[int]:
352
+ """Return the number of a level name, or None with a warning."""
353
+ if level is None or level == "":
354
+ return None
355
+ try:
356
+ return logger.level(str(level).upper()).no
357
+ except ValueError:
358
+ warnings.append(f"Unknown level '{level}' for {what} ignored")
359
+ return None
360
+
361
+
362
+ def _checked_translation(name: str, translation: Any,
363
+ warnings: List[str]) -> Dict[str, Any]:
364
+ """Keep the valid pairs of a translation: level name -> name or False."""
365
+ checked: Dict[str, Any] = {}
366
+ if not isinstance(translation, dict):
367
+ warnings.append(f"translate of logger '{name}' must be a table; "
368
+ f"ignored")
369
+ return checked
370
+ for source, target in translation.items():
371
+ if target is False:
372
+ checked[source.upper()] = False
373
+ elif _level_no(target, f"translation {source} of logger '{name}'",
374
+ warnings) is not None:
375
+ checked[source.upper()] = str(target).upper()
376
+ return checked
377
+
378
+
379
+ def _checked_output(kind: str, params: Dict[str, Any],
380
+ warnings: List[str]) -> Dict[str, Any]:
381
+ """Keep the parameters loguru accepts for this kind of output."""
382
+ allowed = set(inspect.signature(logger.add).parameters) - {"kwargs"}
383
+ allowed -= RESERVED_OUTPUT_KEYS
384
+ allowed |= OWN_OUTPUT_KEYS
385
+ if kind == "file":
386
+ allowed |= FILE_SINK_KEYS
387
+ else:
388
+ allowed -= {"path"}
389
+ checked = {}
390
+ for key, value in params.items():
391
+ if key in allowed:
392
+ checked[key] = value
393
+ else:
394
+ warnings.append(f"Parameter '{key}' of output '{kind}' ignored")
395
+ return checked
396
+
397
+
398
+ def _add_outputs(output, settings: Settings, warnings: List[str]) -> None:
399
+ """Create console and file handlers on the output logger."""
400
+ created = 0
401
+ for kind, params in settings.outputs.items():
402
+ params = dict(params, backtrace=settings.backtrace,
403
+ diagnose=settings.diagnose)
404
+ sink = params.pop("path") if kind == "file" else sys.stderr
405
+ try:
406
+ output.add(sink, level=0, **params)
407
+ created += 1
408
+ except (OSError, TypeError, ValueError) as exc:
409
+ warnings.append(f"Output '{kind}' could not be created ({exc}); "
410
+ f"writing to the console")
411
+ if not created:
412
+ output.add(sys.stderr, level=0)
413
+
414
+
415
+ # --- Levels -----------------------------------------------------------
416
+
417
+ def _threshold(process: str, name: Optional[str]) -> int:
418
+ """Level a message of this process and name must at least have."""
419
+ key = (process, name)
420
+ if key not in _threshold_cache:
421
+ _threshold_cache[key] = _lookup_threshold(process, name or "")
422
+ return _threshold_cache[key]
423
+
424
+
425
+ def _lookup_threshold(process: str, name: str) -> int:
426
+ """Walk the layers, highest first; the first one with a value wins."""
427
+ for layer in _settings.layers:
428
+ entry = _longest_match(layer["loggers"], name, "level")
429
+ if entry is not None:
430
+ return entry
431
+ if process in layer["processes"]:
432
+ return layer["processes"][process]
433
+ if layer["std_level"] is not None:
434
+ return layer["std_level"]
435
+ return 0
436
+
437
+
438
+ def _longest_match(entries: Dict[str, Dict[str, Any]], name: str,
439
+ field: str) -> Any:
440
+ """Return the field of the entry for name or its nearest parent."""
441
+ while True:
442
+ if name in entries and field in entries[name]:
443
+ return entries[name][field]
444
+ if not name:
445
+ return None
446
+ cut = name.rfind(".")
447
+ name = name[:cut] if cut != -1 else ""
448
+ if not name:
449
+ return None
450
+
451
+
452
+ def _min_threshold(process: str) -> int:
453
+ """Lowest level any message of this process could pass with."""
454
+ candidates = []
455
+ for layer in _settings.layers:
456
+ candidates += [entry["level"] for entry in layer["loggers"].values()
457
+ if "level" in entry]
458
+ if process in layer["processes"]:
459
+ candidates.append(layer["processes"][process])
460
+ break
461
+ if layer["std_level"] is not None:
462
+ candidates.append(layer["std_level"])
463
+ break
464
+ return min(candidates, default=0)
465
+
466
+
467
+ def _level_filter(record: Dict[str, Any]) -> bool:
468
+ """loguru filter: pass what reaches the threshold of its origin."""
469
+ return record["level"].no >= _threshold(record["process"].name,
470
+ record["name"])
471
+
472
+
473
+ def _translation(name: str) -> Dict[str, Any]:
474
+ """Translation for a logger: from the highest layer that has one."""
475
+ for layer in _settings.layers:
476
+ found = _longest_match(layer["loggers"], name, "translate")
477
+ if found is not None:
478
+ return found
479
+ return {}
480
+
481
+
482
+ # --- Taking over the standard logging module ----------------------------
483
+
484
+ class _InterceptHandler(logging.Handler):
485
+ """Hands every record of ``logging`` to loguru (loguru's own recipe).
486
+
487
+ The record keeps the logger name as its name, so that the entries
488
+ per logger apply; its level is translated first.
489
+ """
490
+
491
+ def emit(self, record: logging.LogRecord) -> None:
492
+ """Pass one record on, or drop it if its translation says so."""
493
+ target = _translation(record.name).get(record.levelname,
494
+ record.levelname)
495
+ if target is False:
496
+ return
497
+ try:
498
+ level = logger.level(target).name
499
+ except ValueError:
500
+ level = record.levelno
501
+
502
+ # Find the caller outside of logging, so that {function} and
503
+ # {line} point into the library that logged.
504
+ frame, depth = inspect.currentframe(), 0
505
+ while frame is not None and (depth == 0 or
506
+ frame.f_code.co_filename == logging.__file__):
507
+ frame = frame.f_back
508
+ depth += 1
509
+
510
+ logger.patch(lambda r: r.update(name=record.name)).opt(
511
+ depth=depth, exception=record.exc_info).log(
512
+ level, record.getMessage())
513
+
514
+
515
+ def _take_over_std_logging(process: str) -> None:
516
+ """Route ``logging`` of this process into loguru."""
517
+ lowest = _min_threshold(process)
518
+ for layer in _settings.layers:
519
+ for entry in layer["loggers"].values():
520
+ for source, target in entry.get("translate", {}).items():
521
+ if target is not False and logger.level(target).no >= lowest:
522
+ std_no = logging.getLevelName(source)
523
+ if isinstance(std_no, int):
524
+ lowest = min(lowest, std_no)
525
+ logging.basicConfig(handlers=[_InterceptHandler()], level=lowest,
526
+ force=True)
527
+
528
+
529
+ # --- Transport between the processes ------------------------------------
530
+
531
+ def _to_item(message) -> Dict[str, Any]:
532
+ """What travels for one message: its data, not loguru's objects."""
533
+ record = message.record
534
+ return {
535
+ "time": record["time"],
536
+ "level": record["level"].name,
537
+ "process": record["process"].name,
538
+ "pid": record["process"].id,
539
+ "name": record["name"],
540
+ "module": record["module"],
541
+ "function": record["function"],
542
+ "line": record["line"],
543
+ "text": str(message).rstrip("\n"),
544
+ }
545
+
546
+
547
+ class _PipeSink:
548
+ """loguru sink of a child process: sends each message into its pipe.
549
+
550
+ Only this process writes into the pipe, so no lock is shared with
551
+ another process. loguru serialises the calls within the process.
552
+ """
553
+
554
+ def __init__(self, conn):
555
+ self._conn = conn
556
+
557
+ def __call__(self, message) -> None:
558
+ self._conn.send(_to_item(message))
559
+
560
+
561
+ class _Collector(threading.Thread):
562
+ """Main process: gathers all messages and writes them sorted by time.
563
+
564
+ Watches the pipes of the children with multiprocessing.connection.
565
+ wait(), takes the main process's own messages through collect(), and
566
+ writes a message once it is older than the ordering window. A
567
+ message that arrives later than that is written at once.
568
+ """
569
+
570
+ def __init__(self, output, window_s: float):
571
+ super().__init__(name="logdruid-collector", daemon=True)
572
+ self._output = output
573
+ self._window_s = window_s
574
+ self._pipes: Dict[Any, str] = {}
575
+ self._heap: List[tuple] = []
576
+ self._order = itertools.count()
577
+ self._lock = threading.Lock()
578
+ self._stopping = False
579
+
580
+ def add_pipe(self, conn, process_name: str) -> None:
581
+ """Watch the read end of a child's pipe from now on."""
582
+ with self._lock:
583
+ self._pipes[conn] = process_name
584
+
585
+ def collect(self, message) -> None:
586
+ """loguru sink of the main process: queue its own messages."""
587
+ item = _to_item(message)
588
+ if self.is_alive():
589
+ self._push(item)
590
+ else:
591
+ self._write(item)
592
+
593
+ def stop(self, timeout_s: float) -> None:
594
+ """Read the pipes to their end, write everything, then return."""
595
+ self._stopping = True
596
+ self.join(timeout_s)
597
+ if self.is_alive():
598
+ self._output.warning(
599
+ f"Logging pipes of {sorted(self._pipes.values())} did not "
600
+ f"end within {timeout_s} s; their remaining messages are lost")
601
+ with self._lock:
602
+ self._flush(due_only=False)
603
+
604
+ def run(self) -> None:
605
+ """Read, order and write until stop() and the end of every pipe."""
606
+ while True:
607
+ with self._lock:
608
+ pipes = list(self._pipes)
609
+ if self._stopping and not pipes:
610
+ break
611
+ for conn in (wait(pipes, timeout=self._poll_time()) if pipes
612
+ else self._sleep()):
613
+ self._read(conn)
614
+ with self._lock:
615
+ self._flush(due_only=True)
616
+ with self._lock:
617
+ self._flush(due_only=False)
618
+
619
+ def _poll_time(self) -> float:
620
+ """How long to wait: until the next message is due, at most a bit."""
621
+ with self._lock:
622
+ if not self._heap:
623
+ return COLLECT_POLL_S
624
+ due = self._heap[0][0] + self._window_s - time.time()
625
+ return max(0.0, min(due, COLLECT_POLL_S))
626
+
627
+ def _sleep(self) -> list:
628
+ """Wait a little when no pipe is open yet; return nothing to read."""
629
+ time.sleep(COLLECT_POLL_S)
630
+ return []
631
+
632
+ def _read(self, conn) -> None:
633
+ """Take one message from a pipe, or drop the pipe at its end."""
634
+ try:
635
+ item = conn.recv()
636
+ except EOFError:
637
+ self._drop(conn, None)
638
+ return
639
+ except Exception as exc:
640
+ # A child killed while sending leaves half a message behind.
641
+ self._drop(conn, exc)
642
+ return
643
+ self._push(item)
644
+
645
+ def _drop(self, conn, exc: Optional[Exception]) -> None:
646
+ """Stop watching a pipe; report it if it did not end cleanly."""
647
+ with self._lock:
648
+ name = self._pipes.pop(conn, "?")
649
+ conn.close()
650
+ if exc is not None:
651
+ self._output.warning(f"Logging pipe of {name} broke off "
652
+ f"({type(exc).__name__}); later messages "
653
+ f"of that process are lost")
654
+
655
+ def _push(self, item: Dict[str, Any]) -> None:
656
+ """Queue a message by its time stamp."""
657
+ with self._lock:
658
+ heapq.heappush(self._heap, (item["time"].timestamp(),
659
+ next(self._order), item))
660
+ if self._window_s == 0:
661
+ with self._lock:
662
+ self._flush(due_only=False)
663
+
664
+ def _flush(self, due_only: bool) -> None:
665
+ """Write queued messages; with due_only those past the window.
666
+
667
+ The caller holds the lock.
668
+ """
669
+ limit = time.time() - self._window_s
670
+ while self._heap and (not due_only or self._heap[0][0] <= limit):
671
+ _, _, item = heapq.heappop(self._heap)
672
+ self._write(item)
673
+
674
+ def _write(self, item: Dict[str, Any]) -> None:
675
+ """Write one message with the data it arose with."""
676
+
677
+ def restore(record):
678
+ record.update(time=item["time"], name=item["name"],
679
+ module=item["module"], function=item["function"],
680
+ line=item["line"],
681
+ process=type(record["process"])(item["pid"],
682
+ item["process"]))
683
+
684
+ self._output.patch(restore).log(item["level"], item["text"])