omdnotificationforwarder 4.6.1__tar.gz → 4.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/PKG-INFO +47 -1
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/README.md +46 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/bin/notificationforwarder +49 -3
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/pyproject.toml +1 -1
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/baseclass.py +65 -35
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/example/forwarder.py +2 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/json/logger.py +13 -1
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/nostr/forwarder.py +2 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/rabbitmq/forwarder.py +3 -1
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/spool.py +17 -7
- omdnotificationforwarder-4.8/src/notificationforwarder/spool_dump.py +259 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/text/logger.py +44 -7
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/webhook/forwarder.py +2 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/TESTS.md +1 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_builtin_plugins_focus.py +20 -2
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_classes.py +2 -2
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_contracts_forwarders.py +7 -7
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_delivery_resilience.py +4 -3
- omdnotificationforwarder-4.8/tests/test_lint_gate.py +80 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_loggers.py +16 -1
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_nostr.py +2 -2
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_resilience_spool.py +610 -22
- omdnotificationforwarder-4.8/tests/test_spool_dump.py +669 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/.gitignore +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/component_loader.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/email/formatter.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/email/forwarder.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/example/formatter.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/json/__init__.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/naemonlog/reporter.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/nostr/__init__.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/nostr/formatter.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/oauth2webhook/__init__.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/oauth2webhook/forwarder.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/rabbitmq/formatter.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/runtime_config.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/runtime_flow.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/syslog/formatter.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/syslog/forwarder.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/telegram/forwarder.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/text/__init__.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/util.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/lib/python/notificationforwarder/split1/forwarder.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/lib/python/notificationforwarder/split2/formatter.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/lib/python/notificationforwarder/split2/forwarder.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/lib/python/notificationforwarder/split3/formatter.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/lib/python/notificationforwarder/split3/forwarder.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/alertmanager_servicenow/formatter.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/bayern/formatter.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/datadup/formatter.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/datapost/formatter.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/discard/formatter.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/split1/formatter.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/split2/forwarder.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/split3/formatter.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/split3/forwarder.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/split4/formatter.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/split4/forwarder.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/ticketsystem/forwarder.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/ticketsystem/reporter.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/pythonpath/local/lib/python/notificationforwarder/vong/formatter.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_alertmanager.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_contracts_formatters.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_contracts_reporters.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_discard.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_formatter.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_integration_cli.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_loading_resolution.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_logger.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_oauth2webhook.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_orchestration_flow.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_package.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_paths.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_reporter.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_runtime_foundation.py +0 -0
- {omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/tests/test_webhook.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: omdnotificationforwarder
|
|
3
|
-
Version: 4.
|
|
3
|
+
Version: 4.8
|
|
4
4
|
Summary: A framework for notification scripts for OMD
|
|
5
5
|
Project-URL: Homepage, https://github.com/lausser/noteventificationforhandlerwarder
|
|
6
6
|
Project-URL: Bug Tracker, https://github.com/lausser/noteventificationforhandlerwarder/issues
|
|
@@ -490,6 +490,46 @@ For broader confidence, run the full suite:
|
|
|
490
490
|
pytest
|
|
491
491
|
```
|
|
492
492
|
|
|
493
|
+
### Focused lint gate
|
|
494
|
+
|
|
495
|
+
`notificationforwarder/.flake8` selects `E9,E722,F63,F7,F82` and sets
|
|
496
|
+
`max-line-length = 120`. The check fails for undefined names, syntax errors,
|
|
497
|
+
other selected structural errors, and bare `except` handlers.
|
|
498
|
+
Undefined-name detection (F821) stays enabled: it is not suppressed globally,
|
|
499
|
+
per file, or with inline `noqa`. Forwarders whose `logger` is injected by the
|
|
500
|
+
component loader bind `logger = None`; they do not suppress F821.
|
|
501
|
+
|
|
502
|
+
CI runs this gate on Python 3.12 with flake8 7.5.0. The `test` extra lists
|
|
503
|
+
`flake8` without a pin; Python 3.9 and 3.10 cannot install flake8 7.5.0
|
|
504
|
+
(that release requires Python >=3.11), which is why the gate job is pinned
|
|
505
|
+
separately.
|
|
506
|
+
|
|
507
|
+
Run the gate:
|
|
508
|
+
|
|
509
|
+
```bash
|
|
510
|
+
# from notificationforwarder/
|
|
511
|
+
flake8 --config .flake8 src/notificationforwarder --count --statistics
|
|
512
|
+
|
|
513
|
+
# from the repository root
|
|
514
|
+
flake8 --config notificationforwarder/.flake8 notificationforwarder/src/notificationforwarder --count --statistics
|
|
515
|
+
```
|
|
516
|
+
|
|
517
|
+
Expect zero findings on a clean tree.
|
|
518
|
+
|
|
519
|
+
Full diagnostic report, from `notificationforwarder/`. This is not the gate:
|
|
520
|
+
codes it reports that the gate does not select are not certified fixed or
|
|
521
|
+
harmless; reviewers read this report for those findings (examples on the
|
|
522
|
+
current tree: E501, F401, E402):
|
|
523
|
+
|
|
524
|
+
```bash
|
|
525
|
+
flake8 --isolated src/notificationforwarder/ --max-line-length=120 --statistics --count
|
|
526
|
+
```
|
|
527
|
+
|
|
528
|
+
`tests/test_lint_gate.py` is the committed rejection probe: it proves the gate
|
|
529
|
+
rejects an undefined name, fails closed on a missing config, and that the
|
|
530
|
+
`logger = None` bindings do not hide other F821 findings. It runs flake8 from
|
|
531
|
+
the same interpreter that has the `test` extra installed.
|
|
532
|
+
|
|
493
533
|
## Reporters
|
|
494
534
|
|
|
495
535
|
Like *forwarder* and *formatter*, a *reporter* is an instance of a *NotificationReporter* class defined in a file named *reporter.py*. There is one class coming with notificationforwarder, the *NaemonlogReporter*. It's purpose it to write a message to the Naemon logfile. When notificationforwarder is run as a standalone script (and not triggered as a notificationhandler by Naemon), the *NaemonlogReporter* can nevertheless leave a line in the Naemon log.
|
|
@@ -535,6 +575,12 @@ sqlite3 $OMD_ROOT/var/tmp/notificationforwarder_<forwarder>_notifications.db "SE
|
|
|
535
575
|
|
|
536
576
|
`<forwarder>` is the value of `--forwarder`, followed by `_` and the value of `--forwardertag` if one is set (for example `remedy_TEST`).
|
|
537
577
|
|
|
578
|
+
### Correlating Spool Log Entries
|
|
579
|
+
|
|
580
|
+
When a failed delivery is spooled, its `forward failed` line identifies the spool row and includes the stored raw event and its summary, if available. During replay, the delete, stays-in-spool, discard, could-not-format and delete-trash lines likewise include the row id and raw event, plus a non-empty summary when one exists. Failed replays also include any submit error. Every expired row gets its own id and raw-event line alongside the aggregate expired count; an undecodable expired row is logged with its id and payload text.
|
|
581
|
+
|
|
582
|
+
The text logger includes these details in each message, for example `delete spooled event 1 summary <sum: disk full> raw event <{'description': 'disk full'}>`. The JSON logger exposes them in the `msg` object as `event_id`, `raw_event` (the full structured stored event), and `summary` when available. The move-to-end line links the old `event_id` to `new_event_id`; follow that link if an event receives a new spool id before deletion.
|
|
583
|
+
|
|
538
584
|
### Delivery Order
|
|
539
585
|
|
|
540
586
|
Spooled events are stored and re-submitted in the order in which they failed. As long as every delivery succeeds, the receiver gets the events in the order in which they were created. After delivery failures, this is not guaranteed. A newer event can reach the receiver before an older one, for example a RECOVERY before its PROBLEM:
|
|
@@ -469,6 +469,46 @@ For broader confidence, run the full suite:
|
|
|
469
469
|
pytest
|
|
470
470
|
```
|
|
471
471
|
|
|
472
|
+
### Focused lint gate
|
|
473
|
+
|
|
474
|
+
`notificationforwarder/.flake8` selects `E9,E722,F63,F7,F82` and sets
|
|
475
|
+
`max-line-length = 120`. The check fails for undefined names, syntax errors,
|
|
476
|
+
other selected structural errors, and bare `except` handlers.
|
|
477
|
+
Undefined-name detection (F821) stays enabled: it is not suppressed globally,
|
|
478
|
+
per file, or with inline `noqa`. Forwarders whose `logger` is injected by the
|
|
479
|
+
component loader bind `logger = None`; they do not suppress F821.
|
|
480
|
+
|
|
481
|
+
CI runs this gate on Python 3.12 with flake8 7.5.0. The `test` extra lists
|
|
482
|
+
`flake8` without a pin; Python 3.9 and 3.10 cannot install flake8 7.5.0
|
|
483
|
+
(that release requires Python >=3.11), which is why the gate job is pinned
|
|
484
|
+
separately.
|
|
485
|
+
|
|
486
|
+
Run the gate:
|
|
487
|
+
|
|
488
|
+
```bash
|
|
489
|
+
# from notificationforwarder/
|
|
490
|
+
flake8 --config .flake8 src/notificationforwarder --count --statistics
|
|
491
|
+
|
|
492
|
+
# from the repository root
|
|
493
|
+
flake8 --config notificationforwarder/.flake8 notificationforwarder/src/notificationforwarder --count --statistics
|
|
494
|
+
```
|
|
495
|
+
|
|
496
|
+
Expect zero findings on a clean tree.
|
|
497
|
+
|
|
498
|
+
Full diagnostic report, from `notificationforwarder/`. This is not the gate:
|
|
499
|
+
codes it reports that the gate does not select are not certified fixed or
|
|
500
|
+
harmless; reviewers read this report for those findings (examples on the
|
|
501
|
+
current tree: E501, F401, E402):
|
|
502
|
+
|
|
503
|
+
```bash
|
|
504
|
+
flake8 --isolated src/notificationforwarder/ --max-line-length=120 --statistics --count
|
|
505
|
+
```
|
|
506
|
+
|
|
507
|
+
`tests/test_lint_gate.py` is the committed rejection probe: it proves the gate
|
|
508
|
+
rejects an undefined name, fails closed on a missing config, and that the
|
|
509
|
+
`logger = None` bindings do not hide other F821 findings. It runs flake8 from
|
|
510
|
+
the same interpreter that has the `test` extra installed.
|
|
511
|
+
|
|
472
512
|
## Reporters
|
|
473
513
|
|
|
474
514
|
Like *forwarder* and *formatter*, a *reporter* is an instance of a *NotificationReporter* class defined in a file named *reporter.py*. There is one class coming with notificationforwarder, the *NaemonlogReporter*. It's purpose it to write a message to the Naemon logfile. When notificationforwarder is run as a standalone script (and not triggered as a notificationhandler by Naemon), the *NaemonlogReporter* can nevertheless leave a line in the Naemon log.
|
|
@@ -514,6 +554,12 @@ sqlite3 $OMD_ROOT/var/tmp/notificationforwarder_<forwarder>_notifications.db "SE
|
|
|
514
554
|
|
|
515
555
|
`<forwarder>` is the value of `--forwarder`, followed by `_` and the value of `--forwardertag` if one is set (for example `remedy_TEST`).
|
|
516
556
|
|
|
557
|
+
### Correlating Spool Log Entries
|
|
558
|
+
|
|
559
|
+
When a failed delivery is spooled, its `forward failed` line identifies the spool row and includes the stored raw event and its summary, if available. During replay, the delete, stays-in-spool, discard, could-not-format and delete-trash lines likewise include the row id and raw event, plus a non-empty summary when one exists. Failed replays also include any submit error. Every expired row gets its own id and raw-event line alongside the aggregate expired count; an undecodable expired row is logged with its id and payload text.
|
|
560
|
+
|
|
561
|
+
The text logger includes these details in each message, for example `delete spooled event 1 summary <sum: disk full> raw event <{'description': 'disk full'}>`. The JSON logger exposes them in the `msg` object as `event_id`, `raw_event` (the full structured stored event), and `summary` when available. The move-to-end line links the old `event_id` to `new_event_id`; follow that link if an event receives a new spool id before deletion.
|
|
562
|
+
|
|
517
563
|
### Delivery Order
|
|
518
564
|
|
|
519
565
|
Spooled events are stored and re-submitted in the order in which they failed. As long as every delivery succeeds, the receiver gets the events in the order in which they were created. After delivery failures, this is not guaranteed. A newer event can reach the receiver before an older one, for example a RECOVERY before its PROBLEM:
|
|
@@ -8,6 +8,7 @@ import sys
|
|
|
8
8
|
#import logging
|
|
9
9
|
from notificationforwarder.util import setup_logging
|
|
10
10
|
from notificationforwarder import baseclass
|
|
11
|
+
from notificationforwarder import spool_dump
|
|
11
12
|
import traceback
|
|
12
13
|
|
|
13
14
|
|
|
@@ -32,8 +33,9 @@ if __name__ == '__main__':
|
|
|
32
33
|
This can be any backend you want, provided
|
|
33
34
|
there is a Python module forwarder.py in the folder
|
|
34
35
|
notificationforwarder/<forwarder> somewhere in your PYTHONPATH.
|
|
35
|
-
Examples are victorops, syslog, servicenow, rabbitmq,...
|
|
36
|
-
|
|
36
|
+
Examples are victorops, syslog, servicenow, rabbitmq,...
|
|
37
|
+
Defaults to syslog when not in dump mode. Required with --dump-spool.''',
|
|
38
|
+
default=argparse.SUPPRESS)
|
|
37
39
|
parser.add_argument('--forwardertag', '--receivertag', action='store',
|
|
38
40
|
dest="forwardertag",
|
|
39
41
|
help='A distinctive string which is added to the logfile',
|
|
@@ -77,13 +79,57 @@ Example for an HTTP-based reporter:
|
|
|
77
79
|
dest="logger",
|
|
78
80
|
help='Logger type: text (default) or json, or custom logger module',
|
|
79
81
|
default='text')
|
|
82
|
+
parser.add_argument('--dump-spool', action='store_true',
|
|
83
|
+
dest="dump_spool",
|
|
84
|
+
help='''Dump the spooled events of the selected forwarder instance and exit.
|
|
85
|
+
The spool is read only: nothing is forwarded, retried, or modified.
|
|
86
|
+
--forwarder is required in this mode; --forwardertag selects the tagged instance.''',
|
|
87
|
+
default=False)
|
|
88
|
+
parser.add_argument('--order', action='store',
|
|
89
|
+
dest="order",
|
|
90
|
+
choices=("oldest", "newest"),
|
|
91
|
+
help='Spool dump order: oldest (default) or newest',
|
|
92
|
+
default="oldest")
|
|
93
|
+
parser.add_argument('--limit', action='store',
|
|
94
|
+
dest="limit",
|
|
95
|
+
help='Spool dump: maximum number of records to display (default: all)',
|
|
96
|
+
default=None)
|
|
97
|
+
parser.add_argument('--dump-format', action='store',
|
|
98
|
+
dest="dump_format",
|
|
99
|
+
choices=("text", "json"),
|
|
100
|
+
help='Spool dump output format: text (default) or json',
|
|
101
|
+
default="text")
|
|
80
102
|
parser.add_argument('--version', action='version',
|
|
81
|
-
version=f'%(prog)s 4.
|
|
103
|
+
version=f'%(prog)s 4.8')
|
|
82
104
|
|
|
83
105
|
args = parser.parse_args()
|
|
106
|
+
if not hasattr(args, 'forwarder'):
|
|
107
|
+
# The default applies only outside dump mode; dump mode needs the
|
|
108
|
+
# instance named explicitly because the spool is one file per
|
|
109
|
+
# (forwarder, tag) pair.
|
|
110
|
+
if args.dump_spool:
|
|
111
|
+
parser.error("--dump-spool requires --forwarder (or --receiver)")
|
|
112
|
+
args.forwarder = "syslog"
|
|
84
113
|
if not hasattr(args, 'formatter'):
|
|
85
114
|
args.formatter = args.forwarder
|
|
86
115
|
|
|
116
|
+
if args.dump_spool:
|
|
117
|
+
try:
|
|
118
|
+
limit = int(args.limit) if args.limit is not None else None
|
|
119
|
+
except ValueError:
|
|
120
|
+
# one usage line naming the value, like the core's validation
|
|
121
|
+
print("error: --limit must be an integer, got '{}'".format(args.limit),
|
|
122
|
+
file=sys.stderr)
|
|
123
|
+
sys.exit(2)
|
|
124
|
+
status = spool_dump.dump_spool(
|
|
125
|
+
args.forwarder,
|
|
126
|
+
args.forwardertag,
|
|
127
|
+
args.order,
|
|
128
|
+
limit,
|
|
129
|
+
args.dump_format,
|
|
130
|
+
)
|
|
131
|
+
sys.exit(status)
|
|
132
|
+
|
|
87
133
|
try:
|
|
88
134
|
forwarder = baseclass.new(args.forwarder, args.forwardertag, args.formatter, args.verbose, args.debug, args.forwarderopt, args.reporter, args.reporteropt, args.logger)
|
|
89
135
|
except Exception as a:
|
|
@@ -269,13 +269,11 @@ class NotificationForwarder(object):
|
|
|
269
269
|
result = self.forward_formatted(formatted_event)
|
|
270
270
|
success, report_payload, error_message = apply_forward_result(result)
|
|
271
271
|
if not success and not formatted_event.is_heartbeat:
|
|
272
|
-
|
|
273
|
-
if
|
|
274
|
-
failure_context =
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
'status': 'failed',
|
|
278
|
-
}
|
|
272
|
+
spool_success, event_id = self.spool(raw_event)
|
|
273
|
+
if spool_success:
|
|
274
|
+
failure_context = self.spool_event_context(event_id, formatted_event, raw_event)
|
|
275
|
+
failure_context['spooled'] = True
|
|
276
|
+
failure_context['status'] = 'failed'
|
|
279
277
|
if error_message:
|
|
280
278
|
failure_context['exception'] = error_message
|
|
281
279
|
logger.warning("forward failed", failure_context)
|
|
@@ -400,19 +398,38 @@ class NotificationForwarder(object):
|
|
|
400
398
|
|
|
401
399
|
def spool(self, raw_event):
|
|
402
400
|
try:
|
|
403
|
-
self.spool_store.enqueue(raw_event)
|
|
401
|
+
event_id = self.spool_store.enqueue(raw_event)
|
|
404
402
|
spooled_events = self.num_spooled_events()
|
|
405
403
|
logger.warning("spooling queue length", {
|
|
406
404
|
'queue_length': spooled_events
|
|
407
405
|
})
|
|
408
|
-
return True
|
|
406
|
+
return (True, event_id)
|
|
409
407
|
except Exception as e:
|
|
410
408
|
logger.critical("database error", {
|
|
411
409
|
'database_error': True,
|
|
412
410
|
'exception': e
|
|
413
411
|
})
|
|
414
412
|
logger.info("raw event details", {'raw_event': raw_event})
|
|
415
|
-
return False
|
|
413
|
+
return (False, None)
|
|
414
|
+
|
|
415
|
+
def spool_event_context(self, event_id, formatted_event, raw_event):
|
|
416
|
+
"""Build the log context for a line which names a spool entry.
|
|
417
|
+
|
|
418
|
+
The id and the decoded raw event are always included, so every line
|
|
419
|
+
answers by itself which event is behind it. The summary is only
|
|
420
|
+
included when a formatted event exists and has a non-empty summary,
|
|
421
|
+
otherwise the key is omitted. The raw event carries the content even
|
|
422
|
+
when no formatted event exists (formatter failure).
|
|
423
|
+
"""
|
|
424
|
+
context = {
|
|
425
|
+
'event_id': event_id,
|
|
426
|
+
'raw_event': raw_event,
|
|
427
|
+
}
|
|
428
|
+
if formatted_event is not None:
|
|
429
|
+
context['formatted_event'] = formatted_event
|
|
430
|
+
if getattr(formatted_event, 'summary', None):
|
|
431
|
+
context['summary'] = formatted_event.summary
|
|
432
|
+
return context
|
|
416
433
|
|
|
417
434
|
def acquire_lock_with_retry(self, lock_file, max_attempts=3, base_delay=0.1):
|
|
418
435
|
return acquire_lock_with_retry(lock_file, logger, max_attempts=max_attempts, base_delay=base_delay)
|
|
@@ -454,7 +471,8 @@ class NotificationForwarder(object):
|
|
|
454
471
|
if locked:
|
|
455
472
|
self.baseclass_logs_summary = True
|
|
456
473
|
try:
|
|
457
|
-
|
|
474
|
+
dropped_rows = self.spool_store.prune_expired(self.max_spool_minutes)
|
|
475
|
+
dropped = len(dropped_rows)
|
|
458
476
|
replay_attempted = 0
|
|
459
477
|
replayed = 0
|
|
460
478
|
stayed_in_spool = 0
|
|
@@ -464,6 +482,24 @@ class NotificationForwarder(object):
|
|
|
464
482
|
'spooled_count': dropped,
|
|
465
483
|
'action': 'dropped'
|
|
466
484
|
})
|
|
485
|
+
# one line per expired event, so the operator can see
|
|
486
|
+
# which events were thrown away. decoding happens
|
|
487
|
+
# outside the database write lock, wrapped per row,
|
|
488
|
+
# so an undecodable payload cannot abort the flush
|
|
489
|
+
for expired_id, expired_text in dropped_rows:
|
|
490
|
+
try:
|
|
491
|
+
expired_event = self.spool_store.decode(expired_text)
|
|
492
|
+
except Exception as e:
|
|
493
|
+
logger.critical("expired spool entry could not be decoded", {
|
|
494
|
+
'event_id': expired_id,
|
|
495
|
+
'payload_text': expired_text,
|
|
496
|
+
'action': 'expired_undecodable',
|
|
497
|
+
'exception': e,
|
|
498
|
+
})
|
|
499
|
+
continue
|
|
500
|
+
context = self.spool_event_context(expired_id, None, expired_event)
|
|
501
|
+
context['action'] = 'expired'
|
|
502
|
+
logger.info("expired spooled event", context)
|
|
467
503
|
# walk through the rows which existed when the flush started.
|
|
468
504
|
# rows spooled meanwhile by concurrent processes are left for the
|
|
469
505
|
# next flush and each row is tried at most once. the first failed
|
|
@@ -484,13 +520,13 @@ class NotificationForwarder(object):
|
|
|
484
520
|
for id, text in id_events:
|
|
485
521
|
last_id = id
|
|
486
522
|
raw_event = self.spool_store.decode(text)
|
|
487
|
-
|
|
523
|
+
# Keep the stored event intact for logging even if plugins mutate their input.
|
|
524
|
+
formatted_event = self.format_event(self.spool_store.decode(text))
|
|
488
525
|
if formatted_event and formatted_event.is_discarded:
|
|
489
526
|
deleted_trash += 1
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
})
|
|
527
|
+
context = self.spool_event_context(id, formatted_event, raw_event)
|
|
528
|
+
context['action'] = 'discard_during_replay'
|
|
529
|
+
logger.info("discard spooled event during replay", context)
|
|
494
530
|
self.spool_store.delete(id)
|
|
495
531
|
elif formatted_event:
|
|
496
532
|
replay_attempted += 1
|
|
@@ -503,35 +539,29 @@ class NotificationForwarder(object):
|
|
|
503
539
|
if success:
|
|
504
540
|
self.spool_store.delete(id)
|
|
505
541
|
replayed += 1
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
})
|
|
542
|
+
context = self.spool_event_context(id, formatted_event, raw_event)
|
|
543
|
+
context['spooled_count'] = 1
|
|
544
|
+
context['action'] = 'delete'
|
|
545
|
+
logger.info("delete spooled event", context)
|
|
511
546
|
else:
|
|
512
547
|
stayed_in_spool += 1
|
|
513
|
-
context =
|
|
514
|
-
'event_id': id,
|
|
515
|
-
'action': 'stays_in_spool'
|
|
516
|
-
}
|
|
548
|
+
context = self.spool_event_context(id, formatted_event, raw_event)
|
|
517
549
|
if error_message:
|
|
518
550
|
context['exception'] = error_message
|
|
551
|
+
context['action'] = 'stays_in_spool'
|
|
519
552
|
logger.critical("event stays in spool", context)
|
|
520
553
|
self.flush_stopped_at = id
|
|
521
554
|
break
|
|
522
555
|
else:
|
|
523
556
|
deleted_trash += 1
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
'action': 'could_not_format'
|
|
529
|
-
})
|
|
557
|
+
context = self.spool_event_context(id, None, raw_event)
|
|
558
|
+
context['spooled_count'] = 1
|
|
559
|
+
context['action'] = 'could_not_format'
|
|
560
|
+
logger.critical("could not format spooled event", context)
|
|
530
561
|
self.spool_store.delete(id)
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
})
|
|
562
|
+
context = self.spool_event_context(id, None, raw_event)
|
|
563
|
+
context['action'] = 'delete_trash'
|
|
564
|
+
logger.info("delete trash event", context)
|
|
535
565
|
if self.flush_stopped_at is not None:
|
|
536
566
|
# only the rows this flush was responsible for, not
|
|
537
567
|
# the ones spooled concurrently after the snapshot
|
|
@@ -89,7 +89,7 @@ class JsonLogger(NotificationLogger):
|
|
|
89
89
|
msg_data["exception"]["trace"] = ''.join(
|
|
90
90
|
traceback.format_exception(*context['exc_info'])
|
|
91
91
|
)
|
|
92
|
-
except:
|
|
92
|
+
except Exception:
|
|
93
93
|
pass
|
|
94
94
|
else:
|
|
95
95
|
msg_data["exception"] = str(exc)
|
|
@@ -139,6 +139,18 @@ class JsonLogger(NotificationLogger):
|
|
|
139
139
|
msg_data["reporter_class"] = rpt.__class__.__name__
|
|
140
140
|
msg_data["reporter_module"] = rpt.__module_file__
|
|
141
141
|
|
|
142
|
+
# Add the full decoded raw event as its own structured field, even
|
|
143
|
+
# when a formatted event is also present. This is the payload as it
|
|
144
|
+
# was stored in the spool and is not reduced to Naemon keys.
|
|
145
|
+
if 'raw_event' in context and context['raw_event'] is not None:
|
|
146
|
+
msg_data["raw_event"] = context['raw_event']
|
|
147
|
+
# The summary of the spooled event, only present when the formatted
|
|
148
|
+
# event exists and has a non-empty summary
|
|
149
|
+
if 'summary' in context and context['summary']:
|
|
150
|
+
msg_data["summary"] = context['summary']
|
|
151
|
+
if 'payload_text' in context:
|
|
152
|
+
msg_data["payload_text"] = context['payload_text']
|
|
153
|
+
|
|
142
154
|
# Add event data for errors
|
|
143
155
|
if 'event_data' in context:
|
|
144
156
|
msg_data["event_data"] = context['event_data']
|
|
@@ -3,6 +3,8 @@ import json
|
|
|
3
3
|
import logging
|
|
4
4
|
from notificationforwarder.baseclass import NotificationForwarder, NotificationFormatter, timeout
|
|
5
5
|
|
|
6
|
+
logger = None # Set by the component loader.
|
|
7
|
+
|
|
6
8
|
|
|
7
9
|
class RabbitmqForwarder(NotificationForwarder):
|
|
8
10
|
def __init__(self, opts):
|
|
@@ -50,7 +52,7 @@ class RabbitmqForwarder(NotificationForwarder):
|
|
|
50
52
|
logger.critical("rabbitmq connect failed with error {}".format(e))
|
|
51
53
|
return False
|
|
52
54
|
|
|
53
|
-
def disconnect(
|
|
55
|
+
def disconnect(self):
|
|
54
56
|
try:
|
|
55
57
|
self.connection.close()
|
|
56
58
|
except Exception as e:
|
{omdnotificationforwarder-4.6.1 → omdnotificationforwarder-4.8}/src/notificationforwarder/spool.py
RENAMED
|
@@ -45,18 +45,28 @@ class SpoolStore:
|
|
|
45
45
|
sql_insert = "INSERT INTO " + self.table_name + "(payload) VALUES (?)"
|
|
46
46
|
with self._lock:
|
|
47
47
|
self.cursor.execute(sql_insert, (json.dumps(raw_event),))
|
|
48
|
+
# capture the lastrowid inside the lock, a later INSERT on the
|
|
49
|
+
# shared cursor by another thread would overwrite it
|
|
50
|
+
event_id = self.cursor.lastrowid
|
|
48
51
|
self.connection.commit()
|
|
52
|
+
return event_id
|
|
49
53
|
|
|
50
54
|
def prune_expired(self, max_spool_minutes):
|
|
51
55
|
outdated = int(time.time() - 60 * max_spool_minutes)
|
|
52
|
-
|
|
56
|
+
sql_expire = "WHERE CAST(STRFTIME('%s', timestamp) AS INTEGER) < ?"
|
|
53
57
|
with self._lock:
|
|
54
|
-
self.cursor.execute(
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
58
|
+
self.cursor.execute("BEGIN IMMEDIATE")
|
|
59
|
+
try:
|
|
60
|
+
# The transaction prevents another connection from changing
|
|
61
|
+
# the rows between the selection and deletion.
|
|
62
|
+
self.cursor.execute("SELECT id, payload FROM " + self.table_name + " " + sql_expire, (outdated,))
|
|
63
|
+
expired = self.cursor.fetchall()
|
|
64
|
+
self.cursor.execute("DELETE FROM " + self.table_name + " " + sql_expire, (outdated,))
|
|
65
|
+
self.connection.commit()
|
|
66
|
+
return expired
|
|
67
|
+
except Exception:
|
|
68
|
+
self.connection.rollback()
|
|
69
|
+
raise
|
|
60
70
|
|
|
61
71
|
def max_id(self):
|
|
62
72
|
with self._lock:
|