hydra-structlog 0.1.0__py3-none-any.whl

hydra_plugins/hydra_structlog/__init__.py

@@ -0,0 +1,5 @@
+__all__ = [
+    "HydraStructlogSearchPathPlugin",
+]
+
+from hydra_structlog.plugin import HydraStructlogSearchPathPlugin

hydra_structlog/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

hydra_structlog/conf/hydra/job_logging/structlog.yaml

@@ -0,0 +1,102 @@
+# @package hydra.job_logging
+# python logging configuration for tasks
+version: 1
+formatters:
+  console:
+    "()": hydra_structlog.formatters.build_processor_formatter
+    foreign_pre_chain: ${hydra.job_logging.formatters.json.foreign_pre_chain}
+    processors:
+      _target_: hydra_structlog.processors.named_processor_chain
+      remove_processors_meta:
+        _target_: structlog.stdlib.ProcessorFormatter.remove_processors_meta
+        _partial_: true
+      time_stamper:
+        _target_: structlog.processors.TimeStamper
+        fmt: "%Y-%m-%d %H:%M:%S"
+        utc: false
+      console_renderer:
+        _target_: structlog.dev.ConsoleRenderer
+  json:
+    "()": hydra_structlog.formatters.build_processor_formatter
+    foreign_pre_chain:
+      _target_: hydra_structlog.processors.named_processor_chain
+      merge_contextvars:
+        _target_: structlog.contextvars.merge_contextvars
+        _partial_: true
+      add_logger_name:
+        _target_: structlog.stdlib.add_logger_name
+        _partial_: true
+      add_log_level:
+        _target_: structlog.stdlib.add_log_level
+        _partial_: true
+      positional_arguments_formatter:
+        _target_: structlog.stdlib.PositionalArgumentsFormatter
+      extra_adder:
+        _target_: structlog.stdlib.ExtraAdder
+      stack_info_renderer:
+        _target_: structlog.processors.StackInfoRenderer
+    processors:
+      _target_: hydra_structlog.processors.named_processor_chain
+      remove_processors_meta:
+        _target_: structlog.stdlib.ProcessorFormatter.remove_processors_meta
+        _partial_: true
+      static_field_adder:
+        _target_: hydra_structlog.processors.StaticFieldAdder
+        service: ${oc.env:SERVICE,null}
+        version: ${oc.env:VERSION,null}
+        env: ${oc.env:ENV,dev}
+        task: ${hydra.job.name}
+      callsite_parameter_adder:
+        _target_: structlog.processors.CallsiteParameterAdder
+        parameters:
+          - _target_: structlog.processors.CallsiteParameter
+            _args_:
+              - lineno
+          - _target_: structlog.processors.CallsiteParameter
+            _args_:
+              - func_name
+        additional_ignores:
+          - warnings
+          - hydra_structlog
+      time_stamper:
+        _target_: structlog.processors.TimeStamper
+        fmt: iso
+      field_renamer:
+        _target_: hydra_structlog.processors.FieldRenamer
+        level: status
+        event: message
+      format_exc_info:
+        _target_: structlog.processors.format_exc_info
+        _partial_: true
+      json_renderer:
+        _target_: structlog.processors.JSONRenderer
+        sort_keys: true
+handlers:
+  console:
+    class: hydra_structlog.handlers.TqdmStreamHandler
+    formatter: console
+    stream: ext://sys.stdout
+  file:
+    # Run-local copy of the log
+    class: logging.FileHandler
+    formatter: json
+    filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    encoding: utf-8
+  json:
+    # Global structured log for monitoring or similar
+    class: logging.handlers.RotatingFileHandler
+    formatter: json
+    level: INFO
+    # Fixed location for all runs
+    filename: ${hydra.runtime.cwd}/structured.log
+    encoding: utf-8
+    # 1024**2 = 1 MB
+    maxBytes: 1048576
+    # Important for safe rotation (structured.log.1)
+    backupCount: 1
+
+root:
+  level: INFO
+  handlers: [console, file, json]
+
+disable_existing_loggers: false

hydra_structlog/formatters.py

@@ -0,0 +1,12 @@
+from hydra.utils import instantiate
+from structlog.stdlib import ProcessorFormatter
+
+from .init import init_plugin
+
+
+def build_processor_formatter(**kwargs) -> ProcessorFormatter:
+    """Dynamically instantiate the processor chain from plain logging config."""
+    kwargs.setdefault("_target_", "structlog.stdlib.ProcessorFormatter")
+    processor = instantiate(kwargs)
+    init_plugin(processor.foreign_pre_chain)
+    return processor

hydra_structlog/handlers.py

@@ -0,0 +1,34 @@
+import logging
+from typing import Any, TextIO
+
+try:
+    from tqdm.std import tqdm as std_tqdm  # type: ignore[ty:unresolved-import]
+
+    TqdmClass = type[std_tqdm]
+except ImportError:
+    std_tqdm: Any = None
+    TqdmClass = None
+
+
+class TqdmStreamHandler(logging.StreamHandler):
+    """Handler inheriting from StreamHandler to route logging stream through tqdm."""
+
+    def __init__(
+        self,
+        stream: TextIO | None = None,
+        tqdm_class: TqdmClass = std_tqdm,
+    ):
+        super().__init__(stream=stream)
+        self.tqdm_class = tqdm_class
+
+    if std_tqdm is not None:
+
+        def emit(self, record: logging.LogRecord):
+            try:
+                msg = self.format(record)
+                self.tqdm_class.write(msg, end=self.terminator, file=self.stream)  # type: ignore
+                self.flush()
+            except (KeyboardInterrupt, RecursionError, SystemExit):
+                raise
+            except Exception:
+                self.handleError(record)

hydra_structlog/init.py

@@ -0,0 +1,90 @@
+import logging
+import os
+import sys
+import warnings
+from collections.abc import Sequence
+from types import TracebackType
+from typing import Any, TextIO
+
+import structlog
+
+# Original warnings.showwarning and sys.excepthook
+_showwarning = None
+_excepthook = None
+_configured_once = False
+
+
+def warning_logger(
+    message: Warning | str,
+    category: type[Warning],
+    filename: str,
+    lineno: int,
+    file: TextIO | None = None,
+    line: str | None = None,
+) -> None:
+    """Emit Python-native warnings as logging warnings."""
+    frame = sys._getframe(2)
+    module = frame.f_globals.get("__name__")
+    logger = logging.getLogger(module)
+    logger.warning("%s: %s", category.__name__, f"{message} (line {lineno})")
+
+
+def exception_logger(
+    exc_type: type[BaseException],
+    exc_value: BaseException,
+    exc_traceback: TracebackType | None,
+) -> Any:
+    """Exception hook to log uncaught exceptions."""
+    logger = logging.getLogger("exception")
+
+    if issubclass(exc_type, KeyboardInterrupt):
+        logger.error("User interrupt")
+        return
+
+    # Skip Hydra frames in stack trace (see hydra._internal.utils.run_and_report)
+    skip_files = (
+        "hydra/core/utils.py",  # hydra
+        "hydra_zen/wrapper/_implementations.py",  # hydra-zen
+        "hydra/main.py",  # hydra through hydra-zen
+        "pydantic/_internal/_validate_call.py",  # hydra-zen with pydantic wrapper
+    )
+    try:
+        tb = exc_traceback
+        while tb is not None:
+            frame = tb.tb_frame
+            tb = tb.tb_next
+            filename = frame.f_code.co_filename.replace(os.sep, "/")
+            if filename.endswith(skip_files) and tb:
+                exc_traceback = tb
+    except Exception:
+        pass
+
+    message = getattr(exc_value, "message", str(exc_value))
+    exc_info = exc_type, exc_value, exc_traceback
+    logger.critical(message, exc_info=exc_info)
+
+
+def init_plugin(foreign_pre_chain: Sequence[structlog.typing.Processor]) -> None:
+    """Configure structlog and warning and exception hooks."""
+    global _excepthook, _showwarning, _configured_once
+
+    if _showwarning is None:
+        _showwarning = warnings.showwarning
+        warnings.showwarning = warning_logger  # type: ignore[ty:invalid-assignment]
+
+    if _excepthook is None:
+        os.environ["HYDRA_FULL_ERROR"] = "1"  # Suppress Hydra's exception hook
+        _excepthook = sys.excepthook
+        sys.excepthook = exception_logger
+
+    if _configured_once is False:
+        structlog.configure(
+            processors=(
+                list(foreign_pre_chain)
+                + [structlog.stdlib.ProcessorFormatter.wrap_for_formatter]
+            ),
+            logger_factory=structlog.stdlib.LoggerFactory(),
+            wrapper_class=structlog.stdlib.BoundLogger,
+            cache_logger_on_first_use=True,
+        )
+        _configured_once = True

hydra_structlog/plugin.py

@@ -0,0 +1,7 @@
+from hydra.core.config_search_path import ConfigSearchPath
+from hydra.plugins.search_path_plugin import SearchPathPlugin
+
+
+class HydraStructlogSearchPathPlugin(SearchPathPlugin):
+    def manipulate_search_path(self, search_path: ConfigSearchPath) -> None:
+        search_path.append("hydra-structlog", "pkg://hydra_structlog.conf")

hydra_structlog/processors.py

@@ -0,0 +1,43 @@
+from collections.abc import Callable, MutableMapping
+from typing import Any
+
+from structlog.typing import EventDict, WrappedLogger
+
+
+class StaticFieldAdder:
+    """Structlog processor to add static fields to all logs."""
+
+    def __init__(self, **fields: str):
+        self.fields = fields
+
+    def __call__(
+        self, logger: WrappedLogger, name: str, event_dict: EventDict
+    ) -> EventDict:
+        for field, value in self.fields.items():
+            event_dict.setdefault(field, value)
+        return event_dict
+
+
+class FieldRenamer:
+    """Structlog processor to rename fields. Existing fields of the new name are
+    overwritten.
+    """
+
+    def __init__(self, **mapping: str):
+        self.mapping = mapping
+
+    def __call__(
+        self, logger: WrappedLogger, name: str, event_dict: EventDict
+    ) -> EventDict:
+        for field, replace in self.mapping.items():
+            if field in event_dict:
+                event_dict[replace] = event_dict.pop(field)
+        return event_dict
+
+
+Processor = Callable[[Any, str, MutableMapping], MutableMapping]
+
+
+def named_processor_chain(**processors: Processor) -> list[Processor]:
+    """Convert a mapping of named processors into list. Skip None"""
+    return [p for p in processors.values() if p is not None]

hydra_structlog-0.1.0.dist-info/METADATA

@@ -0,0 +1,112 @@
+Metadata-Version: 2.4
+Name: hydra-structlog
+Version: 0.1.0
+Summary: An opinionated hydra plugin to configure logging with structlog
+Keywords: hydra,structlog,plugin,hydra-zen
+Author: Sören Zapp
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Scientific/Engineering
+Requires-Dist: hydra-core>=1.3.2
+Requires-Dist: structlog>=25.5.0
+Requires-Python: >=3.10
+Project-URL: Source, https://github.com/szapp/hydra-structlog
+Project-URL: Issues, https://github.com/szapp/hydra-structlog/issues
+Project-URL: Funding, https://ko-fi.com/szapp
+Description-Content-Type: text/markdown
+
+# Hydra-Structlog
+
+An opinionated hydra plugin to configure logging with structlog.
+
+## Usage
+
+Structlog is fully integrated in Hydra through a config group named `structlog` for `hydra.job_logging`.
+
+1. Install the plugin.
+
+    ```sh
+    uv add hydra-structlog
+    ```
+
+2. Override the job logging with a default in your config.
+
+    ```yaml
+    defaults:
+      - override hydra/job_logging=structlog
+    ```
+
+3. Hydra will now emit JSON logs (by default to `structured.log`) and print human readably logs to the console.
+
+## Customization
+
+Structlog comes configured with opinionated defaults.
+
+To adjust the processors, use the hydra defaults as outlined above and overwrite individual config entries afterwards.
+For example:
+
+```yaml
+defaults:
+  - override hydra/job_logging=structlog
+
+# Change the storage location
+hydra:
+  job_logging:
+    handlers:
+      json:
+        filename: ${hydra.runtime.cwd}/logs/structured.log
+```
+
+## Defaults
+
+The configuration runs structlog through the standard library logging.
+That has several advantages.
+
+The use of structlog and logging, e.g. by third-party packages, are recorded and both exhibit structlog's bound contextvars.
+
+The processor chain is conveniently exposed in the job_logging configuration and adjustable through hydra's config interface.
+
+Because lists are not easily manipulated in hydra configs, the processor chains are defined as dictionaries.
+The name of each processor does not matter as long as its unique in the chain.
+The default processor chain can then be extended more comfortably.
+
+To remove a processor set it to `null`. See example below.
+
+Because structlog's ProcessorFormatter does not accept the processors as strings, the logging config contains `_target_` instantiation.
+
+```yaml
+defaults:
+  - override hydra/job_logging=structlog
+
+# Remove a processor from the default chain and add another one
+hydra:
+  job_logging:
+    formatters:
+      json:
+        foreign_pre_chain:
+          # Remove the add-log-level processor
+          add_logger_name: null
+          # Append a new processor
+          arbitrary_name:
+            # Follow hydra's instantiate syntax
+            _target_: structlog.stdlib.add_log_level_number
+            _partial_: true
+```
+
+## Notes
+
+Python 3.14 is currently not supported as `hydra.main` causes a ValueError on argparse-validation, see <https://github.com/facebookresearch/hydra/issues/3121>.
+It is recommended to use `hydra-zen` which does not have that problem and works well with Python 3.14.
+
+Tracer injection is not implemented here as data science projects using hydra typically do not implement web apps.
+Nevertheless, an appropriate structlog processor can be added to the job_logging config.

hydra_structlog-0.1.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+hydra_plugins/hydra_structlog/__init__.py,sha256=b8nxszR3JwULO_rfmSkOs27b1QwJPM_dOHpmiRTXIP0,119
+hydra_plugins/hydra_structlog/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+hydra_structlog/__init__.py,sha256=kUR5RAFc7HCeiqdlX36dZOHkUI5wI6V_43RpEcD8b-0,22
+hydra_structlog/conf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+hydra_structlog/conf/hydra/job_logging/structlog.yaml,sha256=3U247Abw0ca6wN3END6ZFl8STk1-P7Wa_C6HKrJvBhk,3372
+hydra_structlog/formatters.py,sha256=Tff7FtD9CQFHxhobGMrD2I94R8sldCIaL4Nng_zuvOc,436
+hydra_structlog/handlers.py,sha256=2jR9CNCKPpFBpiN9fcUwza5VvRB-k1Fazyi60_jKVr4,992
+hydra_structlog/init.py,sha256=xgz375XzRfcZ-8CFe4ewCKRp88ShRdnt6ax4zw2do2s,2856
+hydra_structlog/plugin.py,sha256=o0QZHx2Grx8i8_LfskMrh0io4dQurvftKIpk65vuTPU,332
+hydra_structlog/processors.py,sha256=vXW42l39Spaa32YYY2jnVJEs0R2sbsJ7oxo1vCB7nro,1275
+hydra_structlog/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+hydra_structlog-0.1.0.dist-info/licenses/LICENSE,sha256=748QfVuRg6qe0w5Byrj-yYr0coH58yJGP1WinEvDVvQ,1069
+hydra_structlog-0.1.0.dist-info/WHEEL,sha256=M4DeIjVCA49okfALADZoWX5JOGwnmHb-JOpQHtI-1c0,80
+hydra_structlog-0.1.0.dist-info/METADATA,sha256=jCqNB3W-5Xl9JOZJQZZFcjS4ECHOaWSBVTqNDd-tD-8,3803
+hydra_structlog-0.1.0.dist-info/RECORD,,

hydra_structlog-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.11.2
+Root-Is-Purelib: true
+Tag: py3-none-any

hydra_structlog-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026  Sören Zapp
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.