appn-logging 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,66 @@
1
+ # Copyright 2026 Australian Plant Phenomics Network
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Structured logging for APPN software: one line per record on stderr.
5
+
6
+ Lines are JSON by default, for log aggregation, or readable text when
7
+ ``APPN_LOGGING_FORMAT=text`` is set.
8
+
9
+ At the program's entry point, once:
10
+
11
+ from appn_logging import configure_logging, log_context, new_run_id
12
+
13
+ configure_logging()
14
+ with log_context(run_id=new_run_id()):
15
+ run()
16
+
17
+ Everywhere else, plain standard-library logging; anything passed in ``extra=``
18
+ becomes a field in the line:
19
+
20
+ logger = logging.getLogger(__name__)
21
+ logger.info("Fetched page %d", page, extra={"record_count": 100})
22
+
23
+ Importing this package never changes logging configuration. Values under keys
24
+ that look like credentials are replaced with ``[REDACTED]``; that is a safety
25
+ net, not a licence to pass secrets to a logger.
26
+
27
+ The public surface is what ``__all__`` lists. Anything else is internal and
28
+ may change without a major version bump.
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ from importlib.metadata import version as _distribution_version
34
+
35
+ from appn_logging.collectors import collect_records
36
+ from appn_logging.configuration import (
37
+ DEFAULT_QUIET_LOGGERS,
38
+ LOG_FORMATS,
39
+ configure_logging,
40
+ remove_logging_handlers,
41
+ )
42
+ from appn_logging.context import current_log_context, log_context, new_run_id
43
+ from appn_logging.formatters import (
44
+ LOG_SCHEMA_VERSION,
45
+ REDACTED,
46
+ JsonFormatter,
47
+ TextFormatter,
48
+ )
49
+
50
+ __all__ = [
51
+ "DEFAULT_QUIET_LOGGERS",
52
+ "LOG_FORMATS",
53
+ "LOG_SCHEMA_VERSION",
54
+ "REDACTED",
55
+ "JsonFormatter",
56
+ "TextFormatter",
57
+ "collect_records",
58
+ "configure_logging",
59
+ "current_log_context",
60
+ "log_context",
61
+ "new_run_id",
62
+ "remove_logging_handlers",
63
+ ]
64
+
65
+ # Single source of truth is [project] version in pyproject.toml.
66
+ __version__ = _distribution_version("appn-logging")
@@ -0,0 +1,13 @@
1
+ # Copyright 2026 Australian Plant Phenomics Network
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Run ``python -m appn_logging`` as the ``appn-logs`` command."""
5
+
6
+ from __future__ import annotations
7
+
8
+ import sys
9
+
10
+ from appn_logging.cli import main
11
+
12
+ if __name__ == "__main__":
13
+ sys.exit(main())
appn_logging/cli.py ADDED
@@ -0,0 +1,100 @@
1
+ # Copyright 2026 Australian Plant Phenomics Network
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """The ``appn-logs`` command: show JSON log lines as readable text.
5
+
6
+ appn-ingest 2>&1 | appn-logs
7
+ appn-logs --level WARNING run.log
8
+
9
+ Each JSON log line is shown the way ``APPN_LOGGING_FORMAT=text`` would have
10
+ written it. A line that is not a JSON log line is shown unchanged, so the
11
+ command is safe to use on mixed output.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import argparse
17
+ import json
18
+ import logging
19
+ import os
20
+ import sys
21
+ from collections.abc import Iterable, Iterator, Sequence
22
+ from importlib.metadata import version as _distribution_version
23
+ from pathlib import Path
24
+ from typing import TextIO
25
+
26
+ from appn_logging.formatters import format_fields_as_text
27
+
28
+
29
+ def main(arguments: Sequence[str] | None = None) -> int:
30
+ """Run the ``appn-logs`` command and return its exit status.
31
+
32
+ Args:
33
+ arguments: Command-line arguments. Defaults to ``sys.argv[1:]``.
34
+ """
35
+ options = _parser().parse_args(arguments)
36
+ minimum_level = logging.getLevelNamesMapping()[options.level]
37
+ try:
38
+ for line in _lines(options.files, sys.stdin):
39
+ text = _readable(line.rstrip("\n"), minimum_level)
40
+ if text is not None:
41
+ sys.stdout.write(f"{text}\n")
42
+ except BrokenPipeError: # pragma: no cover - covered by the integration test
43
+ # The reader went away (`appn-logs run.log | head`): not an error.
44
+ # Point stdout at nothing so Python's exit-time flush stays quiet.
45
+ os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno())
46
+ except OSError as error:
47
+ sys.stderr.write(f"appn-logs: {error}\n")
48
+ return 1
49
+ return 0
50
+
51
+
52
+ def _parser() -> argparse.ArgumentParser:
53
+ parser = argparse.ArgumentParser(
54
+ prog="appn-logs",
55
+ description=(
56
+ "Show JSON log lines written by appn-logging as readable text. "
57
+ "Reads the named files, or standard input when none are named."
58
+ ),
59
+ )
60
+ parser.add_argument(
61
+ "files", nargs="*", type=Path, metavar="FILE", help="log files to read"
62
+ )
63
+ parser.add_argument(
64
+ "--level",
65
+ default="DEBUG",
66
+ type=str.upper,
67
+ choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
68
+ help="show only lines at this level or above (default: every line)",
69
+ )
70
+ parser.add_argument(
71
+ "--version",
72
+ action="version",
73
+ version=f"%(prog)s {_distribution_version('appn-logging')}",
74
+ )
75
+ return parser
76
+
77
+
78
+ def _lines(files: Iterable[Path], standard_input: TextIO) -> Iterator[str]:
79
+ """Yield every line of ``files``, or of standard input when there are none."""
80
+ files = list(files)
81
+ if not files:
82
+ yield from standard_input
83
+ return
84
+ for path in files:
85
+ with path.open(encoding="utf-8", errors="replace") as file:
86
+ yield from file
87
+
88
+
89
+ def _readable(line: str, minimum_level: int) -> str | None:
90
+ """Return the text to show for ``line``, or None to leave it out."""
91
+ try:
92
+ fields = json.loads(line)
93
+ except ValueError:
94
+ return line
95
+ if not isinstance(fields, dict) or "message" not in fields:
96
+ return line # JSON, but not one of our log lines.
97
+ level = logging.getLevelNamesMapping().get(str(fields.get("level")))
98
+ if level is not None and level < minimum_level:
99
+ return None
100
+ return format_fields_as_text(fields)
@@ -0,0 +1,124 @@
1
+ # Copyright 2026 Australian Plant Phenomics Network
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Collect the log records emitted inside a block of code.
5
+
6
+ Some log records are also wanted somewhere other than the log: the problems to
7
+ list in a report for the user, the warnings that decide an exit code, the
8
+ lines a test asserts on. :func:`collect_records` gathers them without a second
9
+ way of reporting: the code logs as normal, and the caller chooses which
10
+ records to keep.
11
+
12
+ with collect_records(key="issue_type") as issues:
13
+ build_vocabulary()
14
+ write_report(issues)
15
+
16
+ # elsewhere, in build_vocabulary():
17
+ logger.error("Sheet %s has no name column", sheet, extra={"issue_type": "no_name"})
18
+
19
+ Collected records are still written to the log as usual.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import logging
25
+ from collections.abc import Callable, Iterator
26
+ from contextlib import contextmanager
27
+ from contextvars import ContextVar
28
+
29
+ # The collectors open on the current path of execution. A ContextVar, not a
30
+ # global: each thread and each asyncio task sees only the collectors opened on
31
+ # its own path, so two runs collecting at once do not receive each other's
32
+ # records.
33
+ _open_collectors: ContextVar[tuple[object, ...]] = ContextVar(
34
+ "appn_logging_open_collectors", default=()
35
+ )
36
+
37
+
38
+ class _CollectingHandler(logging.Handler):
39
+ """Keep the records that match, when emitted where the collector is open."""
40
+
41
+ def __init__(
42
+ self,
43
+ *,
44
+ level: int,
45
+ key: str | None,
46
+ match: Callable[[logging.LogRecord], bool] | None,
47
+ ) -> None:
48
+ super().__init__(level)
49
+ self.records: list[logging.LogRecord] = []
50
+ self._key = key
51
+ self._match = match
52
+
53
+ def emit(self, record: logging.LogRecord) -> None:
54
+ """Keep ``record`` if this collector is open here and the record matches."""
55
+ if self not in _open_collectors.get():
56
+ return
57
+ if self._key is not None and self._key not in vars(record):
58
+ return
59
+ try:
60
+ if self._match is not None and not self._match(record):
61
+ return
62
+ except Exception:
63
+ # A faulty `match` must not break the logging call that got here.
64
+ self.handleError(record)
65
+ return
66
+ self.records.append(record)
67
+
68
+
69
+ @contextmanager
70
+ def collect_records(
71
+ *,
72
+ key: str | None = None,
73
+ level: str | int = logging.NOTSET,
74
+ match: Callable[[logging.LogRecord], bool] | None = None,
75
+ logger: logging.Logger | str | None = None,
76
+ ) -> Iterator[list[logging.LogRecord]]:
77
+ """Collect the log records emitted inside the block.
78
+
79
+ With no arguments every record is collected. The arguments narrow that
80
+ down, and a record must satisfy all of the ones given.
81
+
82
+ Only records emitted on the current path of execution are collected: the
83
+ code in the block, what it calls, and asyncio tasks it starts. Records from
84
+ other threads, including threads started inside the block, are not.
85
+
86
+ A collector sees a record only if logging lets it through, so a DEBUG
87
+ record is not collected while the logger's level is INFO.
88
+
89
+ Args:
90
+ key: Collect only records that carry this field, set with
91
+ ``extra={key: ...}``. Read the value with ``getattr(record, key)``.
92
+ level: Collect only records at this level or above, as a name
93
+ (``"WARNING"``) or a number.
94
+ match: Collect only records for which this returns true.
95
+ logger: Collect only from this logger and the loggers below it, given
96
+ as a logger or its name. Defaults to every logger.
97
+
98
+ Yields:
99
+ The list of collected records, in the order they were emitted. It
100
+ fills as the block runs and stops filling when the block exits.
101
+
102
+ Raises:
103
+ ValueError: If ``level`` is not a recognised level.
104
+ """
105
+ resolved_level = level if isinstance(level, int) else _level_number(level)
106
+ target = logger if isinstance(logger, logging.Logger) else logging.getLogger(logger)
107
+ handler = _CollectingHandler(level=resolved_level, key=key, match=match)
108
+
109
+ token = _open_collectors.set((*_open_collectors.get(), handler))
110
+ target.addHandler(handler)
111
+ try:
112
+ yield handler.records
113
+ finally:
114
+ target.removeHandler(handler)
115
+ _open_collectors.reset(token)
116
+ handler.close()
117
+
118
+
119
+ def _level_number(level: str) -> int:
120
+ """Return the numeric level for a level name."""
121
+ resolved = logging.getLevelNamesMapping().get(level.upper())
122
+ if resolved is None:
123
+ raise ValueError(f"unknown log level: {level!r}")
124
+ return resolved
@@ -0,0 +1,233 @@
1
+ # Copyright 2026 Australian Plant Phenomics Network
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Install structured logging on the root logger.
5
+
6
+ Only a program's entry point calls these functions. Library code never does:
7
+ it writes ``logger = logging.getLogger(__name__)`` and logs as normal, and the
8
+ program that imports it decides where the lines go.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import logging
14
+ import os
15
+ import sys
16
+ from collections.abc import Callable, Iterable, Mapping
17
+ from pathlib import Path
18
+ from typing import Final, TextIO, TypeVar
19
+
20
+ from appn_logging.formatters import JsonFormatter, TextFormatter
21
+ from appn_logging.uncaught import (
22
+ install_uncaught_exception_logging,
23
+ remove_uncaught_exception_logging,
24
+ )
25
+
26
+ # Libraries that log every request at INFO. Their lines are noise at that level,
27
+ # and a request URL can carry identifiers that do not belong in logs.
28
+ DEFAULT_QUIET_LOGGERS: Final = ("httpx", "httpcore", "urllib3")
29
+
30
+ ENV_LEVEL: Final = "APPN_LOGGING_LEVEL"
31
+ ENV_FORMAT: Final = "APPN_LOGGING_FORMAT"
32
+ ENV_SERVICE: Final = "APPN_LOGGING_SERVICE"
33
+ ENV_ENVIRONMENT: Final = "APPN_LOGGING_ENVIRONMENT"
34
+ ENV_LOGGER_LEVELS: Final = "APPN_LOGGING_LOGGER_LEVELS"
35
+
36
+ LOG_FORMATS: Final = ("json", "text")
37
+
38
+ # Handlers this module installs carry this name, so reconfiguring replaces them
39
+ # without disturbing handlers added by anything else (pytest's, for one).
40
+ _HANDLER_NAME: Final = "appn_logging"
41
+
42
+ # The service name when a program cannot be identified from how it was started
43
+ # (an interactive session, `python -c`).
44
+ _UNNAMED_SERVICE: Final = "python"
45
+
46
+ _T = TypeVar("_T")
47
+
48
+
49
+ def configure_logging(
50
+ *,
51
+ service: str | None = None,
52
+ level: str | int = "INFO",
53
+ log_format: str = "json",
54
+ stream: TextIO | None = None,
55
+ environment: str | None = None,
56
+ version: str | None = None,
57
+ quiet_loggers: Iterable[str] = DEFAULT_QUIET_LOGGERS,
58
+ logger_levels: Mapping[str, str | int] | None = None,
59
+ extra_sensitive_keys: Iterable[str] = (),
60
+ environ: Mapping[str, str] | None = None,
61
+ ) -> None:
62
+ """Send log records to ``stream`` (stderr by default), one per line.
63
+
64
+ Call once, at the program's entry point. Every argument is optional, so
65
+ ``configure_logging()`` on its own is a complete setup. Calling again
66
+ replaces what the previous call installed and leaves every other handler
67
+ alone.
68
+
69
+ The ``APPN_LOGGING_*`` environment variables are read on every call and
70
+ **win over the arguments**, so whoever runs a program can change its
71
+ logging without editing it. The arguments are the program's defaults.
72
+
73
+ * ``APPN_LOGGING_SERVICE`` overrides ``service``.
74
+ * ``APPN_LOGGING_LEVEL`` overrides ``level``.
75
+ * ``APPN_LOGGING_FORMAT`` overrides ``log_format``.
76
+ * ``APPN_LOGGING_ENVIRONMENT`` overrides ``environment``.
77
+ * ``APPN_LOGGING_LOGGER_LEVELS`` adds to ``logger_levels``, as
78
+ comma-separated ``name=LEVEL`` pairs, e.g.
79
+ ``appn_ingest.fetch=DEBUG,botocore=ERROR``.
80
+
81
+ An unset or empty variable is ignored.
82
+
83
+ Besides writing log records, this sends uncaught exceptions and Python
84
+ warnings to the log, so that a crash is a log line like any other.
85
+
86
+ stdout is left for the program's actual output, so a CLI can be piped
87
+ without log lines mixed into the data.
88
+
89
+ Args:
90
+ service: Name of the program, added to every line as ``service``.
91
+ Defaults to the name the program was started with.
92
+ level: Root logger level, as a name (``"INFO"``) or a number.
93
+ log_format: ``"json"`` for one JSON object per line, which is what log
94
+ aggregation reads, or ``"text"`` for lines a person can read on a
95
+ terminal.
96
+ stream: Destination. Defaults to ``sys.stderr``.
97
+ environment: Deployment environment, e.g. ``"production"``. Omitted
98
+ from the output when not given.
99
+ version: Version of the program. Omitted from the output when not given.
100
+ quiet_loggers: Loggers raised to WARNING because they are noisy at INFO.
101
+ logger_levels: Levels for individual loggers, by logger name, e.g.
102
+ ``{"appn_ingest.fetch": "DEBUG"}``. A level applies to that logger
103
+ and everything below it, and wins over ``level`` and
104
+ ``quiet_loggers``.
105
+ extra_sensitive_keys: Additional key names whose values are redacted.
106
+ environ: Environment variables to read. Defaults to ``os.environ``.
107
+
108
+ Raises:
109
+ ValueError: If a level or the format is not a recognised value. When
110
+ the value came from an environment variable the message names the
111
+ variable. Nothing is changed in that case.
112
+ """
113
+ env = os.environ if environ is None else environ
114
+
115
+ # Resolve and validate everything before touching the logging configuration.
116
+ resolved_service = env.get(ENV_SERVICE) or service or _program_name()
117
+ resolved_level = _from_env(env, ENV_LEVEL, _resolve_level)
118
+ if resolved_level is None:
119
+ resolved_level = _resolve_level(level)
120
+ resolved_format = _from_env(env, ENV_FORMAT, _resolve_format)
121
+ if resolved_format is None:
122
+ resolved_format = _resolve_format(log_format)
123
+ resolved_logger_levels = {
124
+ **{name: _resolve_level(lvl) for name, lvl in (logger_levels or {}).items()},
125
+ **(_from_env(env, ENV_LOGGER_LEVELS, _parse_logger_levels) or {}),
126
+ }
127
+
128
+ formatter_class = TextFormatter if resolved_format == "text" else JsonFormatter
129
+ handler = logging.StreamHandler(stream or sys.stderr)
130
+ handler.set_name(_HANDLER_NAME)
131
+ handler.setFormatter(
132
+ formatter_class(
133
+ service=resolved_service,
134
+ environment=env.get(ENV_ENVIRONMENT) or environment,
135
+ version=version,
136
+ extra_sensitive_keys=extra_sensitive_keys,
137
+ )
138
+ )
139
+
140
+ root = logging.getLogger()
141
+ remove_logging_handlers()
142
+ root.addHandler(handler)
143
+ root.setLevel(resolved_level)
144
+ for name in quiet_loggers:
145
+ logging.getLogger(name).setLevel(logging.WARNING)
146
+ for name, logger_level in resolved_logger_levels.items():
147
+ logging.getLogger(name).setLevel(logger_level)
148
+ install_uncaught_exception_logging()
149
+ logging.captureWarnings(True)
150
+
151
+
152
+ def remove_logging_handlers() -> None:
153
+ """Undo :func:`configure_logging`.
154
+
155
+ Removes the handler it installed, if any, and stops sending uncaught
156
+ exceptions and Python warnings to the log. Logger levels are left as they
157
+ are.
158
+ """
159
+ root = logging.getLogger()
160
+ installed = [h for h in root.handlers if h.get_name() == _HANDLER_NAME]
161
+ for existing in installed:
162
+ root.removeHandler(existing)
163
+ existing.close()
164
+ if installed:
165
+ remove_uncaught_exception_logging()
166
+ logging.captureWarnings(False)
167
+
168
+
169
+ def _from_env(
170
+ env: Mapping[str, str], variable: str, parse: Callable[[str], _T]
171
+ ) -> _T | None:
172
+ """Return the parsed value of ``variable``, or None if it is unset or empty.
173
+
174
+ Raises:
175
+ ValueError: If the value cannot be parsed. The message names the variable.
176
+ """
177
+ value = env.get(variable)
178
+ if not value:
179
+ return None
180
+ try:
181
+ return parse(value)
182
+ except ValueError as error:
183
+ raise ValueError(f"{variable}: {error}") from None
184
+
185
+
186
+ def _program_name() -> str:
187
+ """Return a service name taken from how the program was started."""
188
+ program = Path(sys.argv[0]) if sys.argv and sys.argv[0] else Path()
189
+ # `python -m package` runs package/__main__.py: the package is the name.
190
+ name = program.parent.name if program.stem == "__main__" else program.stem
191
+ return _UNNAMED_SERVICE if not name or name.startswith("-") else name
192
+
193
+
194
+ def _parse_logger_levels(value: str) -> dict[str, int]:
195
+ """Parse ``name=LEVEL,name=LEVEL`` into a mapping of logger name to level.
196
+
197
+ Spaces around names, levels and commas are ignored, and so are empty
198
+ entries, so a trailing comma is harmless.
199
+
200
+ Raises:
201
+ ValueError: If an entry is not ``name=LEVEL`` or its level is unknown.
202
+ """
203
+ logger_levels: dict[str, int] = {}
204
+ for entry in filter(None, (part.strip() for part in value.split(","))):
205
+ name, separator, level = (piece.strip() for piece in entry.partition("="))
206
+ if not (name and separator and level):
207
+ raise ValueError(f"entries must look like name=LEVEL, got {entry!r}")
208
+ try:
209
+ logger_levels[name] = _resolve_level(level)
210
+ except ValueError as error:
211
+ raise ValueError(f"{error} for logger {name!r}") from None
212
+ return logger_levels
213
+
214
+
215
+ def _resolve_level(level: str | int) -> int:
216
+ """Return the numeric level for a level name or number."""
217
+ if isinstance(level, int):
218
+ return level
219
+ resolved = logging.getLevelNamesMapping().get(level.strip().upper())
220
+ if resolved is None:
221
+ raise ValueError(f"unknown log level: {level!r}")
222
+ return resolved
223
+
224
+
225
+ def _resolve_format(log_format: str) -> str:
226
+ """Return ``log_format`` in lower case, checked against :data:`LOG_FORMATS`."""
227
+ resolved = log_format.strip().lower()
228
+ if resolved not in LOG_FORMATS:
229
+ raise ValueError(
230
+ f"unknown log format: {log_format!r} (expected one of: "
231
+ f"{', '.join(LOG_FORMATS)})"
232
+ )
233
+ return resolved
@@ -0,0 +1,53 @@
1
+ # Copyright 2026 Australian Plant Phenomics Network
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Fields attached to every log line inside a block of code.
5
+
6
+ Bind a run ID (or a request ID, a dataset ID, ...) once with
7
+ :func:`log_context` and every line logged inside the block carries it, which is
8
+ how one run is traced through the logs of a program handling several at once.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import uuid
14
+ from collections.abc import Iterator, Mapping
15
+ from contextlib import contextmanager
16
+ from contextvars import ContextVar
17
+ from types import MappingProxyType
18
+
19
+ # A ContextVar, not a global: each thread and each asyncio task sees only the
20
+ # fields bound on its own path, so concurrent runs cannot leak into each other.
21
+ _context: ContextVar[Mapping[str, object]] = ContextVar(
22
+ "appn_logging_context", default=MappingProxyType({})
23
+ )
24
+
25
+
26
+ @contextmanager
27
+ def log_context(**fields: object) -> Iterator[None]:
28
+ """Add ``fields`` to every log line emitted inside the block.
29
+
30
+ Blocks nest: an inner block adds to (or overrides) the outer block's
31
+ fields, and they are restored when the inner block exits.
32
+
33
+ Args:
34
+ **fields: Key-value pairs to attach, e.g. ``run_id=new_run_id()``.
35
+
36
+ Yields:
37
+ Nothing; the fields are removed again when the block exits.
38
+ """
39
+ token = _context.set(MappingProxyType({**_context.get(), **fields}))
40
+ try:
41
+ yield
42
+ finally:
43
+ _context.reset(token)
44
+
45
+
46
+ def current_log_context() -> Mapping[str, object]:
47
+ """Return a read-only view of the fields bound by :func:`log_context`."""
48
+ return _context.get()
49
+
50
+
51
+ def new_run_id() -> str:
52
+ """Return a new random identifier for the ``run_id`` field (a UUID4)."""
53
+ return str(uuid.uuid4())