omdnotificationforwarder 4.6.1__tar.gz → 4.8__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/PKG-INFO +47 -1
  2. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/README.md +46 -0
  3. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/bin/notificationforwarder +49 -3
  4. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/pyproject.toml +1 -1
  5. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/baseclass.py +65 -35
  6. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/example/forwarder.py +2 -0
  7. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/json/logger.py +13 -1
  8. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/nostr/forwarder.py +2 -0
  9. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/rabbitmq/forwarder.py +3 -1
  10. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/spool.py +17 -7
  11. omdnotificationforwarder-4.8/src/notificationforwarder/spool_dump.py +259 -0
  12. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/text/logger.py +44 -7
  13. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/webhook/forwarder.py +2 -0
  14. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/TESTS.md +1 -0
  15. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_builtin_plugins_focus.py +20 -2
  16. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_classes.py +2 -2
  17. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_contracts_forwarders.py +7 -7
  18. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_delivery_resilience.py +4 -3
  19. omdnotificationforwarder-4.8/tests/test_lint_gate.py +80 -0
  20. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_loggers.py +16 -1
  21. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_nostr.py +2 -2
  22. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_resilience_spool.py +610 -22
  23. omdnotificationforwarder-4.8/tests/test_spool_dump.py +669 -0
  24. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/.gitignore +0 -0
  25. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/component_loader.py +0 -0
  26. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/email/formatter.py +0 -0
  27. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/email/forwarder.py +0 -0
  28. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/example/formatter.py +0 -0
  29. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/json/__init__.py +0 -0
  30. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/naemonlog/reporter.py +0 -0
  31. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/nostr/__init__.py +0 -0
  32. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/nostr/formatter.py +0 -0
  33. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/oauth2webhook/__init__.py +0 -0
  34. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/oauth2webhook/forwarder.py +0 -0
  35. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/rabbitmq/formatter.py +0 -0
  36. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/runtime_config.py +0 -0
  37. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/runtime_flow.py +0 -0
  38. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/syslog/formatter.py +0 -0
  39. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/syslog/forwarder.py +0 -0
  40. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/telegram/forwarder.py +0 -0
  41. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/text/__init__.py +0 -0
  42. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/util.py +0 -0
  43. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/lib/python/notificationforwarder/split1/forwarder.py +0 -0
  44. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/lib/python/notificationforwarder/split2/formatter.py +0 -0
  45. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/lib/python/notificationforwarder/split2/forwarder.py +0 -0
  46. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/lib/python/notificationforwarder/split3/formatter.py +0 -0
  47. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/lib/python/notificationforwarder/split3/forwarder.py +0 -0
  48. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/alertmanager_servicenow/formatter.py +0 -0
  49. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/bayern/formatter.py +0 -0
  50. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/datadup/formatter.py +0 -0
  51. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/datapost/formatter.py +0 -0
  52. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/discard/formatter.py +0 -0
  53. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/split1/formatter.py +0 -0
  54. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/split2/forwarder.py +0 -0
  55. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/split3/formatter.py +0 -0
  56. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/split3/forwarder.py +0 -0
  57. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/split4/formatter.py +0 -0
  58. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/split4/forwarder.py +0 -0
  59. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/ticketsystem/forwarder.py +0 -0
  60. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/ticketsystem/reporter.py +0 -0
  61. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/vong/formatter.py +0 -0
  62. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_alertmanager.py +0 -0
  63. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_contracts_formatters.py +0 -0
  64. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_contracts_reporters.py +0 -0
  65. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_discard.py +0 -0
  66. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_formatter.py +0 -0
  67. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_integration_cli.py +0 -0
  68. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_loading_resolution.py +0 -0
  69. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_logger.py +0 -0
  70. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_oauth2webhook.py +0 -0
  71. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_orchestration_flow.py +0 -0
  72. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_package.py +0 -0
  73. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_paths.py +0 -0
  74. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_reporter.py +0 -0
  75. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_runtime_foundation.py +0 -0
  76. {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_webhook.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: omdnotificationforwarder
3
- Version: 4.6.1
3
+ Version: 4.8
4
4
  Summary: A framework for notification scripts for OMD
5
5
  Project-URL: Homepage, https://github.com/lausser/noteventificationforhandlerwarder
6
6
  Project-URL: Bug Tracker, https://github.com/lausser/noteventificationforhandlerwarder/issues
@@ -490,6 +490,46 @@ For broader confidence, run the full suite:
490
490
  pytest
491
491
  ```
492
492
 
493
+ ### Focused lint gate
494
+
495
+ `notificationforwarder/.flake8` selects `E9,E722,F63,F7,F82` and sets
496
+ `max-line-length = 120`. The check fails for undefined names, syntax errors,
497
+ other selected structural errors, and bare `except` handlers.
498
+ Undefined-name detection (F821) stays enabled: it is not suppressed globally,
499
+ per file, or with inline `noqa`. Forwarders whose `logger` is injected by the
500
+ component loader bind `logger = None`; they do not suppress F821.
501
+
502
+ CI runs this gate on Python 3.12 with flake8 7.5.0. The `test` extra lists
503
+ `flake8` without a pin; Python 3.9 and 3.10 cannot install flake8 7.5.0
504
+ (that release requires Python >=3.11), which is why the gate job is pinned
505
+ separately.
506
+
507
+ Run the gate:
508
+
509
+ ```bash
510
+ # from notificationforwarder/
511
+ flake8 --config .flake8 src/notificationforwarder --count --statistics
512
+
513
+ # from the repository root
514
+ flake8 --config notificationforwarder/.flake8 notificationforwarder/src/notificationforwarder --count --statistics
515
+ ```
516
+
517
+ Expect zero findings on a clean tree.
518
+
519
+ Full diagnostic report, from `notificationforwarder/`. This is not the gate:
520
+ codes it reports that the gate does not select are not certified fixed or
521
+ harmless; reviewers read this report for those findings (examples on the
522
+ current tree: E501, F401, E402):
523
+
524
+ ```bash
525
+ flake8 --isolated src/notificationforwarder/ --max-line-length=120 --statistics --count
526
+ ```
527
+
528
+ `tests/test_lint_gate.py` is the committed rejection probe: it proves the gate
529
+ rejects an undefined name, fails closed on a missing config, and that the
530
+ `logger = None` bindings do not hide other F821 findings. It runs flake8 from
531
+ the same interpreter that has the `test` extra installed.
532
+
493
533
  ## Reporters
494
534
 
495
535
  Like *forwarder* and *formatter*, a *reporter* is an instance of a *NotificationReporter* class defined in a file named *reporter.py*. There is one class coming with notificationforwarder, the *NaemonlogReporter*. It's purpose it to write a message to the Naemon logfile. When notificationforwarder is run as a standalone script (and not triggered as a notificationhandler by Naemon), the *NaemonlogReporter* can nevertheless leave a line in the Naemon log.
@@ -535,6 +575,12 @@ sqlite3 $OMD_ROOT/var/tmp/notificationforwarder_<forwarder>_notifications.db "SE
535
575
 
536
576
  `<forwarder>` is the value of `--forwarder`, followed by `_` and the value of `--forwardertag` if one is set (for example `remedy_TEST`).
537
577
 
578
+ ### Correlating Spool Log Entries
579
+
580
+ When a failed delivery is spooled, its `forward failed` line identifies the spool row and includes the stored raw event and its summary, if available. During replay, the delete, stays-in-spool, discard, could-not-format and delete-trash lines likewise include the row id and raw event, plus a non-empty summary when one exists. Failed replays also include any submit error. Every expired row gets its own id and raw-event line alongside the aggregate expired count; an undecodable expired row is logged with its id and payload text.
581
+
582
+ The text logger includes these details in each message, for example `delete spooled event 1 summary <sum: disk full> raw event <{'description': 'disk full'}>`. The JSON logger exposes them in the `msg` object as `event_id`, `raw_event` (the full structured stored event), and `summary` when available. The move-to-end line links the old `event_id` to `new_event_id`; follow that link if an event receives a new spool id before deletion.
583
+
538
584
  ### Delivery Order
539
585
 
540
586
  Spooled events are stored and re-submitted in the order in which they failed. As long as every delivery succeeds, the receiver gets the events in the order in which they were created. After delivery failures, this is not guaranteed. A newer event can reach the receiver before an older one, for example a RECOVERY before its PROBLEM:
@@ -469,6 +469,46 @@ For broader confidence, run the full suite:
469
469
  pytest
470
470
  ```
471
471
 
472
+ ### Focused lint gate
473
+
474
+ `notificationforwarder/.flake8` selects `E9,E722,F63,F7,F82` and sets
475
+ `max-line-length = 120`. The check fails for undefined names, syntax errors,
476
+ other selected structural errors, and bare `except` handlers.
477
+ Undefined-name detection (F821) stays enabled: it is not suppressed globally,
478
+ per file, or with inline `noqa`. Forwarders whose `logger` is injected by the
479
+ component loader bind `logger = None`; they do not suppress F821.
480
+
481
+ CI runs this gate on Python 3.12 with flake8 7.5.0. The `test` extra lists
482
+ `flake8` without a pin; Python 3.9 and 3.10 cannot install flake8 7.5.0
483
+ (that release requires Python >=3.11), which is why the gate job is pinned
484
+ separately.
485
+
486
+ Run the gate:
487
+
488
+ ```bash
489
+ # from notificationforwarder/
490
+ flake8 --config .flake8 src/notificationforwarder --count --statistics
491
+
492
+ # from the repository root
493
+ flake8 --config notificationforwarder/.flake8 notificationforwarder/src/notificationforwarder --count --statistics
494
+ ```
495
+
496
+ Expect zero findings on a clean tree.
497
+
498
+ Full diagnostic report, from `notificationforwarder/`. This is not the gate:
499
+ codes it reports that the gate does not select are not certified fixed or
500
+ harmless; reviewers read this report for those findings (examples on the
501
+ current tree: E501, F401, E402):
502
+
503
+ ```bash
504
+ flake8 --isolated src/notificationforwarder/ --max-line-length=120 --statistics --count
505
+ ```
506
+
507
+ `tests/test_lint_gate.py` is the committed rejection probe: it proves the gate
508
+ rejects an undefined name, fails closed on a missing config, and that the
509
+ `logger = None` bindings do not hide other F821 findings. It runs flake8 from
510
+ the same interpreter that has the `test` extra installed.
511
+
472
512
  ## Reporters
473
513
 
474
514
  Like *forwarder* and *formatter*, a *reporter* is an instance of a *NotificationReporter* class defined in a file named *reporter.py*. There is one class coming with notificationforwarder, the *NaemonlogReporter*. It's purpose it to write a message to the Naemon logfile. When notificationforwarder is run as a standalone script (and not triggered as a notificationhandler by Naemon), the *NaemonlogReporter* can nevertheless leave a line in the Naemon log.
@@ -514,6 +554,12 @@ sqlite3 $OMD_ROOT/var/tmp/notificationforwarder_<forwarder>_notifications.db "SE
514
554
 
515
555
  `<forwarder>` is the value of `--forwarder`, followed by `_` and the value of `--forwardertag` if one is set (for example `remedy_TEST`).
516
556
 
557
+ ### Correlating Spool Log Entries
558
+
559
+ When a failed delivery is spooled, its `forward failed` line identifies the spool row and includes the stored raw event and its summary, if available. During replay, the delete, stays-in-spool, discard, could-not-format and delete-trash lines likewise include the row id and raw event, plus a non-empty summary when one exists. Failed replays also include any submit error. Every expired row gets its own id and raw-event line alongside the aggregate expired count; an undecodable expired row is logged with its id and payload text.
560
+
561
+ The text logger includes these details in each message, for example `delete spooled event 1 summary <sum: disk full> raw event <{'description': 'disk full'}>`. The JSON logger exposes them in the `msg` object as `event_id`, `raw_event` (the full structured stored event), and `summary` when available. The move-to-end line links the old `event_id` to `new_event_id`; follow that link if an event receives a new spool id before deletion.
562
+
517
563
  ### Delivery Order
518
564
 
519
565
  Spooled events are stored and re-submitted in the order in which they failed. As long as every delivery succeeds, the receiver gets the events in the order in which they were created. After delivery failures, this is not guaranteed. A newer event can reach the receiver before an older one, for example a RECOVERY before its PROBLEM:
@@ -8,6 +8,7 @@ import sys
8
8
  #import logging
9
9
  from notificationforwarder.util import setup_logging
10
10
  from notificationforwarder import baseclass
11
+ from notificationforwarder import spool_dump
11
12
  import traceback
12
13
 
13
14
 
@@ -32,8 +33,9 @@ if __name__ == '__main__':
32
33
  This can be any backend you want, provided
33
34
  there is a Python module forwarder.py in the folder
34
35
  notificationforwarder/<forwarder> somewhere in your PYTHONPATH.
35
- Examples are victorops, syslog, servicenow, rabbitmq,...''',
36
- default="syslog")
36
+ Examples are victorops, syslog, servicenow, rabbitmq,...
37
+ Defaults to syslog when not in dump mode. Required with --dump-spool.''',
38
+ default=argparse.SUPPRESS)
37
39
  parser.add_argument('--forwardertag', '--receivertag', action='store',
38
40
  dest="forwardertag",
39
41
  help='A distinctive string which is added to the logfile',
@@ -77,13 +79,57 @@ Example for an HTTP-based reporter:
77
79
  dest="logger",
78
80
  help='Logger type: text (default) or json, or custom logger module',
79
81
  default='text')
82
+ parser.add_argument('--dump-spool', action='store_true',
83
+ dest="dump_spool",
84
+ help='''Dump the spooled events of the selected forwarder instance and exit.
85
+ The spool is read only: nothing is forwarded, retried, or modified.
86
+ --forwarder is required in this mode; --forwardertag selects the tagged instance.''',
87
+ default=False)
88
+ parser.add_argument('--order', action='store',
89
+ dest="order",
90
+ choices=("oldest", "newest"),
91
+ help='Spool dump order: oldest (default) or newest',
92
+ default="oldest")
93
+ parser.add_argument('--limit', action='store',
94
+ dest="limit",
95
+ help='Spool dump: maximum number of records to display (default: all)',
96
+ default=None)
97
+ parser.add_argument('--dump-format', action='store',
98
+ dest="dump_format",
99
+ choices=("text", "json"),
100
+ help='Spool dump output format: text (default) or json',
101
+ default="text")
80
102
  parser.add_argument('--version', action='version',
81
- version=f'%(prog)s 4.6.1')
103
+ version=f'%(prog)s 4.8')
82
104
 
83
105
  args = parser.parse_args()
106
+ if not hasattr(args, 'forwarder'):
107
+ # The default applies only outside dump mode; dump mode needs the
108
+ # instance named explicitly because the spool is one file per
109
+ # (forwarder, tag) pair.
110
+ if args.dump_spool:
111
+ parser.error("--dump-spool requires --forwarder (or --receiver)")
112
+ args.forwarder = "syslog"
84
113
  if not hasattr(args, 'formatter'):
85
114
  args.formatter = args.forwarder
86
115
 
116
+ if args.dump_spool:
117
+ try:
118
+ limit = int(args.limit) if args.limit is not None else None
119
+ except ValueError:
120
+ # one usage line naming the value, like the core's validation
121
+ print("error: --limit must be an integer, got '{}'".format(args.limit),
122
+ file=sys.stderr)
123
+ sys.exit(2)
124
+ status = spool_dump.dump_spool(
125
+ args.forwarder,
126
+ args.forwardertag,
127
+ args.order,
128
+ limit,
129
+ args.dump_format,
130
+ )
131
+ sys.exit(status)
132
+
87
133
  try:
88
134
  forwarder = baseclass.new(args.forwarder, args.forwardertag, args.formatter, args.verbose, args.debug, args.forwarderopt, args.reporter, args.reporteropt, args.logger)
89
135
  except Exception as a:
@@ -21,7 +21,7 @@ packages = ["src/notificationforwarder"]
21
21
 
22
22
  [project]
23
23
  name = "omdnotificationforwarder"
24
- version = "4.6.1"
24
+ version = "4.8"
25
25
  authors = [
26
26
  { name="Gerhard Lausser", email="lausser@yahoo.com" },
27
27
  ]
@@ -269,13 +269,11 @@ class NotificationForwarder(object):
269
269
  result = self.forward_formatted(formatted_event)
270
270
  success, report_payload, error_message = apply_forward_result(result)
271
271
  if not success and not formatted_event.is_heartbeat:
272
- spooled = self.spool(raw_event)
273
- if spooled:
274
- failure_context = {
275
- 'formatted_event': formatted_event,
276
- 'spooled': True,
277
- 'status': 'failed',
278
- }
272
+ spool_success, event_id = self.spool(raw_event)
273
+ if spool_success:
274
+ failure_context = self.spool_event_context(event_id, formatted_event, raw_event)
275
+ failure_context['spooled'] = True
276
+ failure_context['status'] = 'failed'
279
277
  if error_message:
280
278
  failure_context['exception'] = error_message
281
279
  logger.warning("forward failed", failure_context)
@@ -400,19 +398,38 @@ class NotificationForwarder(object):
400
398
 
401
399
  def spool(self, raw_event):
402
400
  try:
403
- self.spool_store.enqueue(raw_event)
401
+ event_id = self.spool_store.enqueue(raw_event)
404
402
  spooled_events = self.num_spooled_events()
405
403
  logger.warning("spooling queue length", {
406
404
  'queue_length': spooled_events
407
405
  })
408
- return True
406
+ return (True, event_id)
409
407
  except Exception as e:
410
408
  logger.critical("database error", {
411
409
  'database_error': True,
412
410
  'exception': e
413
411
  })
414
412
  logger.info("raw event details", {'raw_event': raw_event})
415
- return False
413
+ return (False, None)
414
+
415
+ def spool_event_context(self, event_id, formatted_event, raw_event):
416
+ """Build the log context for a line which names a spool entry.
417
+
418
+ The id and the decoded raw event are always included, so every line
419
+ answers by itself which event is behind it. The summary is only
420
+ included when a formatted event exists and has a non-empty summary,
421
+ otherwise the key is omitted. The raw event carries the content even
422
+ when no formatted event exists (formatter failure).
423
+ """
424
+ context = {
425
+ 'event_id': event_id,
426
+ 'raw_event': raw_event,
427
+ }
428
+ if formatted_event is not None:
429
+ context['formatted_event'] = formatted_event
430
+ if getattr(formatted_event, 'summary', None):
431
+ context['summary'] = formatted_event.summary
432
+ return context
416
433
 
417
434
  def acquire_lock_with_retry(self, lock_file, max_attempts=3, base_delay=0.1):
418
435
  return acquire_lock_with_retry(lock_file, logger, max_attempts=max_attempts, base_delay=base_delay)
@@ -454,7 +471,8 @@ class NotificationForwarder(object):
454
471
  if locked:
455
472
  self.baseclass_logs_summary = True
456
473
  try:
457
- dropped = self.spool_store.prune_expired(self.max_spool_minutes)
474
+ dropped_rows = self.spool_store.prune_expired(self.max_spool_minutes)
475
+ dropped = len(dropped_rows)
458
476
  replay_attempted = 0
459
477
  replayed = 0
460
478
  stayed_in_spool = 0
@@ -464,6 +482,24 @@ class NotificationForwarder(object):
464
482
  'spooled_count': dropped,
465
483
  'action': 'dropped'
466
484
  })
485
+ # one line per expired event, so the operator can see
486
+ # which events were thrown away. decoding happens
487
+ # outside the database write lock, wrapped per row,
488
+ # so an undecodable payload cannot abort the flush
489
+ for expired_id, expired_text in dropped_rows:
490
+ try:
491
+ expired_event = self.spool_store.decode(expired_text)
492
+ except Exception as e:
493
+ logger.critical("expired spool entry could not be decoded", {
494
+ 'event_id': expired_id,
495
+ 'payload_text': expired_text,
496
+ 'action': 'expired_undecodable',
497
+ 'exception': e,
498
+ })
499
+ continue
500
+ context = self.spool_event_context(expired_id, None, expired_event)
501
+ context['action'] = 'expired'
502
+ logger.info("expired spooled event", context)
467
503
  # walk through the rows which existed when the flush started.
468
504
  # rows spooled meanwhile by concurrent processes are left for the
469
505
  # next flush and each row is tried at most once. the first failed
@@ -484,13 +520,13 @@ class NotificationForwarder(object):
484
520
  for id, text in id_events:
485
521
  last_id = id
486
522
  raw_event = self.spool_store.decode(text)
487
- formatted_event = self.format_event(raw_event)
523
+ # Keep the stored event intact for logging even if plugins mutate their input.
524
+ formatted_event = self.format_event(self.spool_store.decode(text))
488
525
  if formatted_event and formatted_event.is_discarded:
489
526
  deleted_trash += 1
490
- logger.info("discard spooled event during replay", {
491
- 'event_id': id,
492
- 'action': 'discard_during_replay'
493
- })
527
+ context = self.spool_event_context(id, formatted_event, raw_event)
528
+ context['action'] = 'discard_during_replay'
529
+ logger.info("discard spooled event during replay", context)
494
530
  self.spool_store.delete(id)
495
531
  elif formatted_event:
496
532
  replay_attempted += 1
@@ -503,35 +539,29 @@ class NotificationForwarder(object):
503
539
  if success:
504
540
  self.spool_store.delete(id)
505
541
  replayed += 1
506
- logger.info("delete spooled event", {
507
- 'spooled_count': 1,
508
- 'event_id': id,
509
- 'action': 'delete'
510
- })
542
+ context = self.spool_event_context(id, formatted_event, raw_event)
543
+ context['spooled_count'] = 1
544
+ context['action'] = 'delete'
545
+ logger.info("delete spooled event", context)
511
546
  else:
512
547
  stayed_in_spool += 1
513
- context = {
514
- 'event_id': id,
515
- 'action': 'stays_in_spool'
516
- }
548
+ context = self.spool_event_context(id, formatted_event, raw_event)
517
549
  if error_message:
518
550
  context['exception'] = error_message
551
+ context['action'] = 'stays_in_spool'
519
552
  logger.critical("event stays in spool", context)
520
553
  self.flush_stopped_at = id
521
554
  break
522
555
  else:
523
556
  deleted_trash += 1
524
- logger.critical("could not format spooled event", {
525
- 'raw_event': raw_event,
526
- 'event_id': id,
527
- 'spooled_count': 1,
528
- 'action': 'could_not_format'
529
- })
557
+ context = self.spool_event_context(id, None, raw_event)
558
+ context['spooled_count'] = 1
559
+ context['action'] = 'could_not_format'
560
+ logger.critical("could not format spooled event", context)
530
561
  self.spool_store.delete(id)
531
- logger.info("delete trash event", {
532
- 'event_id': id,
533
- 'action': 'delete_trash'
534
- })
562
+ context = self.spool_event_context(id, None, raw_event)
563
+ context['action'] = 'delete_trash'
564
+ logger.info("delete trash event", context)
535
565
  if self.flush_stopped_at is not None:
536
566
  # only the rows this flush was responsible for, not
537
567
  # the ones spooled concurrently after the snapshot
@@ -3,6 +3,8 @@ import time
3
3
  import logging
4
4
  from notificationforwarder.baseclass import NotificationForwarder, NotificationFormatter, timeout
5
5
 
6
+ logger = None # Set by the component loader.
7
+
6
8
 
7
9
  class ExampleForwarder(NotificationForwarder):
8
10
  def __init__(self, opts):
@@ -89,7 +89,7 @@ class JsonLogger(NotificationLogger):
89
89
  msg_data["exception"]["trace"] = ''.join(
90
90
  traceback.format_exception(*context['exc_info'])
91
91
  )
92
- except:
92
+ except Exception:
93
93
  pass
94
94
  else:
95
95
  msg_data["exception"] = str(exc)
@@ -139,6 +139,18 @@ class JsonLogger(NotificationLogger):
139
139
  msg_data["reporter_class"] = rpt.__class__.__name__
140
140
  msg_data["reporter_module"] = rpt.__module_file__
141
141
 
142
+ # Add the full decoded raw event as its own structured field, even
143
+ # when a formatted event is also present. This is the payload as it
144
+ # was stored in the spool and is not reduced to Naemon keys.
145
+ if 'raw_event' in context and context['raw_event'] is not None:
146
+ msg_data["raw_event"] = context['raw_event']
147
+ # The summary of the spooled event, only present when the formatted
148
+ # event exists and has a non-empty summary
149
+ if 'summary' in context and context['summary']:
150
+ msg_data["summary"] = context['summary']
151
+ if 'payload_text' in context:
152
+ msg_data["payload_text"] = context['payload_text']
153
+
142
154
  # Add event data for errors
143
155
  if 'event_data' in context:
144
156
  msg_data["event_data"] = context['event_data']
@@ -7,6 +7,8 @@ except ImportError as exc:
7
7
 
8
8
  from notificationforwarder.baseclass import NotificationForwarder, timeout
9
9
 
10
+ logger = None # Set by the component loader.
11
+
10
12
 
11
13
  class NostrForwarder(NotificationForwarder):
12
14
  def __init__(self, opts):
@@ -3,6 +3,8 @@ import json
3
3
  import logging
4
4
  from notificationforwarder.baseclass import NotificationForwarder, NotificationFormatter, timeout
5
5
 
6
+ logger = None # Set by the component loader.
7
+
6
8
 
7
9
  class RabbitmqForwarder(NotificationForwarder):
8
10
  def __init__(self, opts):
@@ -50,7 +52,7 @@ class RabbitmqForwarder(NotificationForwarder):
50
52
  logger.critical("rabbitmq connect failed with error {}".format(e))
51
53
  return False
52
54
 
53
- def disconnect(iself):
55
+ def disconnect(self):
54
56
  try:
55
57
  self.connection.close()
56
58
  except Exception as e:
@@ -45,18 +45,28 @@ class SpoolStore:
45
45
  sql_insert = "INSERT INTO " + self.table_name + "(payload) VALUES (?)"
46
46
  with self._lock:
47
47
  self.cursor.execute(sql_insert, (json.dumps(raw_event),))
48
+ # capture the lastrowid inside the lock, a later INSERT on the
49
+ # shared cursor by another thread would overwrite it
50
+ event_id = self.cursor.lastrowid
48
51
  self.connection.commit()
52
+ return event_id
49
53
 
50
54
  def prune_expired(self, max_spool_minutes):
51
55
  outdated = int(time.time() - 60 * max_spool_minutes)
52
- sql_delete = "DELETE FROM " + self.table_name + " WHERE CAST(STRFTIME('%s', timestamp) AS INTEGER) < ?"
56
+ sql_expire = "WHERE CAST(STRFTIME('%s', timestamp) AS INTEGER) < ?"
53
57
  with self._lock:
54
- self.cursor.execute(sql_delete, (outdated,))
55
- dropped = self.cursor.rowcount
56
- # commit immediately, otherwise the implicit transaction keeps
57
- # the database write-locked for other processes during a flush
58
- self.connection.commit()
59
- return dropped
58
+ self.cursor.execute("BEGIN IMMEDIATE")
59
+ try:
60
+ # The transaction prevents another connection from changing
61
+ # the rows between the selection and deletion.
62
+ self.cursor.execute("SELECT id, payload FROM " + self.table_name + " " + sql_expire, (outdated,))
63
+ expired = self.cursor.fetchall()
64
+ self.cursor.execute("DELETE FROM " + self.table_name + " " + sql_expire, (outdated,))
65
+ self.connection.commit()
66
+ return expired
67
+ except Exception:
68
+ self.connection.rollback()
69
+ raise
60
70
 
61
71
  def max_id(self):
62
72
  with self._lock: