sap-ecs-log-forwarder 1.0.2__py3-none-any.whl → 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sap_ecs_log_forwarder/cli.py +115 -2
- sap_ecs_log_forwarder/processor.py +2 -0
- sap_ecs_log_forwarder/processor_sentinel.py +230 -0
- {sap_ecs_log_forwarder-1.0.2.dist-info → sap_ecs_log_forwarder-1.1.0.dist-info}/METADATA +58 -3
- {sap_ecs_log_forwarder-1.0.2.dist-info → sap_ecs_log_forwarder-1.1.0.dist-info}/RECORD +8 -7
- {sap_ecs_log_forwarder-1.0.2.dist-info → sap_ecs_log_forwarder-1.1.0.dist-info}/LICENSE +0 -0
- {sap_ecs_log_forwarder-1.0.2.dist-info → sap_ecs_log_forwarder-1.1.0.dist-info}/WHEEL +0 -0
- {sap_ecs_log_forwarder-1.0.2.dist-info → sap_ecs_log_forwarder-1.1.0.dist-info}/entry_points.txt +0 -0
sap_ecs_log_forwarder/cli.py
CHANGED
|
@@ -80,6 +80,49 @@ def list_inputs():
|
|
|
80
80
|
for i in cfg["inputs"]:
|
|
81
81
|
click.echo(f"- {i['name']} [{i['provider']}]")
|
|
82
82
|
|
|
83
|
+
@input.command("show")
|
|
84
|
+
@click.argument("name")
|
|
85
|
+
def show_input(name):
|
|
86
|
+
"""Show details of a specific input including its outputs."""
|
|
87
|
+
cfg = _load_mutable()
|
|
88
|
+
inp = _find(cfg, name)
|
|
89
|
+
if not inp:
|
|
90
|
+
click.echo("Input not found.")
|
|
91
|
+
return
|
|
92
|
+
click.echo(f"Input: {inp['name']}")
|
|
93
|
+
click.echo(f" Provider: {inp.get('provider', '-')}")
|
|
94
|
+
if inp.get("queue"):
|
|
95
|
+
click.echo(f" Queue: {inp['queue']}")
|
|
96
|
+
if inp.get("subscription"):
|
|
97
|
+
click.echo(f" Subscription: {inp['subscription']}")
|
|
98
|
+
if inp.get("region"):
|
|
99
|
+
click.echo(f" Region: {inp['region']}")
|
|
100
|
+
if inp.get("bucket"):
|
|
101
|
+
click.echo(f" Bucket: {inp['bucket']}")
|
|
102
|
+
if inp.get("storageAccount"):
|
|
103
|
+
click.echo(f" Storage Account: {inp['storageAccount']}")
|
|
104
|
+
click.echo(f" Max Retries: {inp.get('maxRetries', '-')}")
|
|
105
|
+
click.echo(f" Retry Delay: {inp.get('retryDelay', '-')}")
|
|
106
|
+
click.echo(f" Log Level: {inp.get('logLevel', '-')}")
|
|
107
|
+
inc = ", ".join(inp.get("includeFilter", [])) or "-"
|
|
108
|
+
exc = ", ".join(inp.get("excludeFilter", [])) or "-"
|
|
109
|
+
click.echo(f" Include Filter: {inc}")
|
|
110
|
+
click.echo(f" Exclude Filter: {exc}")
|
|
111
|
+
click.echo(f" Authentication: {'configured' if inp.get('authentication') else 'not configured'}")
|
|
112
|
+
outs = inp.get("outputs", [])
|
|
113
|
+
if not outs:
|
|
114
|
+
click.echo(" Outputs: none")
|
|
115
|
+
else:
|
|
116
|
+
click.echo(" Outputs:")
|
|
117
|
+
for idx, o in enumerate(outs):
|
|
118
|
+
dest = o.get("destination", "")
|
|
119
|
+
out_inc = ", ".join(o.get("includeFilter", [])) or "-"
|
|
120
|
+
out_exc = ", ".join(o.get("excludeFilter", [])) or "-"
|
|
121
|
+
if o["type"] == "sentinel":
|
|
122
|
+
click.echo(f" [{idx}] {o['type']} -> {o.get('sentinel_dce_log_ingestion_url', '')} (include: {out_inc}; exclude: {out_exc})")
|
|
123
|
+
else:
|
|
124
|
+
click.echo(f" [{idx}] {o['type']} -> {dest} (include: {out_inc}; exclude: {out_exc})")
|
|
125
|
+
|
|
83
126
|
@input.command("remove")
|
|
84
127
|
@click.argument("name")
|
|
85
128
|
def remove_input(name):
|
|
@@ -98,12 +141,19 @@ def output():
|
|
|
98
141
|
|
|
99
142
|
@output.command("add")
|
|
100
143
|
@click.option("--input-name", prompt=True)
|
|
101
|
-
@click.option("--type", "otype", type=click.Choice(["files","http","console"]), prompt=True)
|
|
144
|
+
@click.option("--type", "otype", type=click.Choice(["files","http","console","sentinel"]), prompt=True)
|
|
102
145
|
@click.option("--destination", help="For files/http")
|
|
103
146
|
@click.option("--compress", is_flag=True, default=False)
|
|
104
147
|
@click.option("--include", "include_filters", multiple=True, help="Regex include filter(s) for output (can be repeated)")
|
|
105
148
|
@click.option("--exclude", "exclude_filters", multiple=True, help="Regex exclude filter(s) for output (can be repeated)")
|
|
106
|
-
|
|
149
|
+
@click.option("--sentinel-dce-tenant-id", help="Sentinel DCE tenant ID")
|
|
150
|
+
@click.option("--sentinel-dce-app-id", help="Sentinel DCE application/client ID")
|
|
151
|
+
@click.option("--sentinel-dce-app-secret", help="Sentinel DCE application secret")
|
|
152
|
+
@click.option("--sentinel-dce-ingestion-url", help="Sentinel DCE log ingestion URL")
|
|
153
|
+
@click.option("--sentinel-dce-dcr-immutable-id", help="Sentinel DCE DCR immutable ID")
|
|
154
|
+
def add_output(input_name, otype, destination, compress, include_filters, exclude_filters,
|
|
155
|
+
sentinel_dce_tenant_id, sentinel_dce_app_id, sentinel_dce_app_secret, sentinel_dce_ingestion_url,
|
|
156
|
+
sentinel_dce_dcr_immutable_id):
|
|
107
157
|
cfg = _load_mutable()
|
|
108
158
|
inp = _find(cfg, input_name)
|
|
109
159
|
if not inp:
|
|
@@ -114,6 +164,17 @@ def add_output(input_name, otype, destination, compress, include_filters, exclud
|
|
|
114
164
|
out["destination"] = destination or click.prompt("Destination")
|
|
115
165
|
if otype == "files":
|
|
116
166
|
out["compress"] = compress
|
|
167
|
+
if otype == "sentinel":
|
|
168
|
+
key = get_active_key()
|
|
169
|
+
if not key:
|
|
170
|
+
click.echo("Encryption key not set (env FORWARDER_ENCRYPTION_KEY).")
|
|
171
|
+
return
|
|
172
|
+
out["sentinel_dce_tenant_id"] = "enc:" + encrypt_value(sentinel_dce_tenant_id or click.prompt("Sentinel DCE tenant ID"), key)
|
|
173
|
+
out["sentinel_dce_application_id"] = "enc:" + encrypt_value(sentinel_dce_app_id or click.prompt("Sentinel DCE application/client ID"), key)
|
|
174
|
+
out["sentinel_dce_application_secret"] = "enc:" + encrypt_value(sentinel_dce_app_secret or click.prompt("Sentinel DCE application secret", hide_input=True), key)
|
|
175
|
+
out["sentinel_dce_log_ingestion_url"] = sentinel_dce_ingestion_url or click.prompt("Sentinel DCE log ingestion URL")
|
|
176
|
+
out["sentinel_dce_dcr_immutable_id"] = sentinel_dce_dcr_immutable_id or click.prompt("Sentinel DCE DCR immutable ID")
|
|
177
|
+
out["sentinel_dce_dcr_stream_id"] = "Custom-SAPLogServ_CL"
|
|
117
178
|
|
|
118
179
|
# Attach output-level filters if provided
|
|
119
180
|
if include_filters:
|
|
@@ -124,6 +185,17 @@ def add_output(input_name, otype, destination, compress, include_filters, exclud
|
|
|
124
185
|
if otype in ("files","http") and not out.get("destination"):
|
|
125
186
|
click.echo("Destination required.")
|
|
126
187
|
return
|
|
188
|
+
|
|
189
|
+
if otype == "sentinel":
|
|
190
|
+
missing = []
|
|
191
|
+
for field in ["sentinel_dce_tenant_id", "sentinel_dce_application_id", "sentinel_dce_application_secret",
|
|
192
|
+
"sentinel_dce_log_ingestion_url", "sentinel_dce_dcr_immutable_id", "sentinel_dce_dcr_stream_id"]:
|
|
193
|
+
if not out.get(field):
|
|
194
|
+
missing.append(field)
|
|
195
|
+
if missing:
|
|
196
|
+
click.echo(f"Missing required sentinel fields: {', '.join(missing)}")
|
|
197
|
+
return
|
|
198
|
+
|
|
127
199
|
inp.setdefault("outputs", []).append(out)
|
|
128
200
|
save_config(cfg)
|
|
129
201
|
click.echo("Output added.")
|
|
@@ -145,6 +217,47 @@ def list_outputs(input_name):
|
|
|
145
217
|
exc = ", ".join(o.get("excludeFilter", [])) or "-"
|
|
146
218
|
click.echo(f"[{idx}] {o['type']} -> {o.get('destination','')} (include: {inc}; exclude: {exc})")
|
|
147
219
|
|
|
220
|
+
@output.command("show")
|
|
221
|
+
@click.option("--input-name", prompt=True)
|
|
222
|
+
@click.option("--index", type=int, prompt=True)
|
|
223
|
+
def show_output(input_name, index):
|
|
224
|
+
"""Show details of a specific output by input name and index."""
|
|
225
|
+
cfg = _load_mutable()
|
|
226
|
+
inp = _find(cfg, input_name)
|
|
227
|
+
if not inp:
|
|
228
|
+
click.echo("Input not found.")
|
|
229
|
+
return
|
|
230
|
+
outs = inp.get("outputs", [])
|
|
231
|
+
if not (0 <= index < len(outs)):
|
|
232
|
+
click.echo("Invalid index.")
|
|
233
|
+
return
|
|
234
|
+
o = outs[index]
|
|
235
|
+
click.echo(f"Output [{index}]:")
|
|
236
|
+
click.echo(f" Type: {o.get('type', '-')}")
|
|
237
|
+
if o.get("destination"):
|
|
238
|
+
click.echo(f" Destination: {o['destination']}")
|
|
239
|
+
if o.get("compress"):
|
|
240
|
+
click.echo(f" Compress: {o['compress']}")
|
|
241
|
+
if o.get("type") == "sentinel":
|
|
242
|
+
click.echo(f" Tenant ID: {o.get('sentinel_dce_tenant_id', '-')}")
|
|
243
|
+
click.echo(f" Application ID: {o.get('sentinel_dce_application_id', '-')}")
|
|
244
|
+
click.echo(f" Log Ingestion URL: {o.get('sentinel_dce_log_ingestion_url', '-')}")
|
|
245
|
+
click.echo(f" DCR Immutable ID: {o.get('sentinel_dce_dcr_immutable_id', '-')}")
|
|
246
|
+
click.echo(f" DCR Stream ID: {o.get('sentinel_dce_dcr_stream_id', '-')}")
|
|
247
|
+
if o.get("authorization"):
|
|
248
|
+
auth = o["authorization"]
|
|
249
|
+
click.echo(f" Authorization: {auth.get('type', '-')} ({'encrypted' if auth.get('encrypted') else 'plain'})")
|
|
250
|
+
if o.get("tls"):
|
|
251
|
+
tls = o["tls"]
|
|
252
|
+
click.echo(f" TLS Client Cert: {tls.get('pathToClientCert', '-')}")
|
|
253
|
+
click.echo(f" TLS Client Key: {tls.get('pathToClientKey', '-')}")
|
|
254
|
+
click.echo(f" TLS CA Cert: {tls.get('pathToCACert', '-')}")
|
|
255
|
+
click.echo(f" TLS Insecure Skip Verify: {tls.get('insecureSkipVerify', False)}")
|
|
256
|
+
inc = ", ".join(o.get("includeFilter", [])) or "-"
|
|
257
|
+
exc = ", ".join(o.get("excludeFilter", [])) or "-"
|
|
258
|
+
click.echo(f" Include Filter: {inc}")
|
|
259
|
+
click.echo(f" Exclude Filter: {exc}")
|
|
260
|
+
|
|
148
261
|
@output.command("remove")
|
|
149
262
|
@click.option("--input-name", prompt=True)
|
|
150
263
|
@click.option("--index", type=int, prompt=True)
|
|
@@ -4,6 +4,7 @@ import requests
|
|
|
4
4
|
import gzip
|
|
5
5
|
|
|
6
6
|
from sap_ecs_log_forwarder import metrics
|
|
7
|
+
from sap_ecs_log_forwarder.processor_sentinel import send_sentinel_dce
|
|
7
8
|
from sap_ecs_log_forwarder.crypto import decrypt_value_if_needed
|
|
8
9
|
from sap_ecs_log_forwarder.utils import compile_filters, is_relevant
|
|
9
10
|
|
|
@@ -84,6 +85,7 @@ OUTPUT_HANDLERS = {
|
|
|
84
85
|
"files": write_file,
|
|
85
86
|
"http": send_http,
|
|
86
87
|
"console": send_console,
|
|
88
|
+
"sentinel": send_sentinel_dce,
|
|
87
89
|
}
|
|
88
90
|
|
|
89
91
|
def emit(lines, source_name, outputs):
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Sentinel Log Ingestion Processor
|
|
3
|
+
This module handles sending log data to Azure Monitor / Microsoft Sentinel
|
|
4
|
+
via the Log Ingestion API.
|
|
5
|
+
Note: This implementation is based on the C# implementation in [JsonProcessor.cs](https://github.com/ulkeba/json-to-law_function/blob/v0.2/JsonProcessor.cs)
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
import logging
|
|
10
|
+
import os
|
|
11
|
+
from datetime import datetime, timedelta, timezone
|
|
12
|
+
from typing import Iterable, Optional
|
|
13
|
+
|
|
14
|
+
import requests
|
|
15
|
+
from azure.identity import ClientSecretCredential, DefaultAzureCredential
|
|
16
|
+
from azure.core.credentials import AccessToken
|
|
17
|
+
|
|
18
|
+
from sap_ecs_log_forwarder import metrics
|
|
19
|
+
from sap_ecs_log_forwarder.crypto import decrypt_value_if_needed
|
|
20
|
+
|
|
21
|
+
class SentinelProcessor:
|
|
22
|
+
"""
|
|
23
|
+
Processes and sends log lines to a Data Colletion Endpoint, to make them
|
|
24
|
+
available through Azure Monitor / Microsoft Sentinel.
|
|
25
|
+
|
|
26
|
+
Supports authentication via:
|
|
27
|
+
- Client credentials (tenant_id, client_id, client_secret)
|
|
28
|
+
- DefaultAzureCredential (e.g., managed identity, Azure CLI login, etc.)
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
MONITOR_SCOPE = "https://monitor.azure.com//.default"
|
|
32
|
+
DEFAULT_BATCH_SIZE = 90
|
|
33
|
+
TOKEN_REFRESH_BUFFER_MINUTES = 5
|
|
34
|
+
|
|
35
|
+
# Cached token (class-level to persist across instances)
|
|
36
|
+
_monitor_token: Optional[AccessToken] = None
|
|
37
|
+
|
|
38
|
+
def __init__(self, cfg: dict, logger: Optional[logging.Logger] = None):
|
|
39
|
+
"""
|
|
40
|
+
Initialize the Sentinel processor.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
cfg: Configuration dictionary with keys:
|
|
44
|
+
- sentinel_dce_tenant_id: Azure AD tenant ID
|
|
45
|
+
- sentinel_dce_application_id: Azure AD client/application ID
|
|
46
|
+
- sentinel_dce_application_secret: Azure AD client secret
|
|
47
|
+
- sentinel_dce_log_ingestion_url: Azure Monitor Data Collection Endpoint URL
|
|
48
|
+
- sentinel_dce_dcr_immutable_id: Data Collection Rule immutable ID
|
|
49
|
+
- sentinel_dce_dcr_stream_id: Data Collection Rule stream ID
|
|
50
|
+
- batch_size: Number of lines per batch (default: 90)
|
|
51
|
+
logger: Optional logger instance
|
|
52
|
+
"""
|
|
53
|
+
self.cfg = cfg
|
|
54
|
+
self.log = logger or logging.getLogger(__name__)
|
|
55
|
+
|
|
56
|
+
# Load configuration from cfg or environment variables
|
|
57
|
+
self._sentinel_dce_tenant_id = cfg.get("sentinel_dce_tenant_id")
|
|
58
|
+
self._sentinel_dce_application_id = cfg.get("sentinel_dce_application_id")
|
|
59
|
+
self._sentinel_dce_application_secret = cfg.get("sentinel_dce_application_secret")
|
|
60
|
+
self._sentinel_dce_log_ingestion_url = cfg.get("sentinel_dce_log_ingestion_url")
|
|
61
|
+
self._sentinel_dce_dcr_immutable_id = cfg.get("sentinel_dce_dcr_immutable_id")
|
|
62
|
+
self._sentinel_dce_dcr_stream_id = cfg.get("sentinel_dce_dcr_stream_id")
|
|
63
|
+
|
|
64
|
+
# Validate required configuration
|
|
65
|
+
required_fields = {
|
|
66
|
+
"sentinel_dce_tenant_id": self._sentinel_dce_tenant_id,
|
|
67
|
+
"sentinel_dce_application_id": self._sentinel_dce_application_id,
|
|
68
|
+
"sentinel_dce_application_secret": self._sentinel_dce_application_secret,
|
|
69
|
+
"sentinel_dce_log_ingestion_url": self._sentinel_dce_log_ingestion_url,
|
|
70
|
+
"sentinel_dce_dcr_immutable_id": self._sentinel_dce_dcr_immutable_id,
|
|
71
|
+
"sentinel_dce_dcr_stream_id": self._sentinel_dce_dcr_stream_id,
|
|
72
|
+
}
|
|
73
|
+
missing_fields = [name for name, value in required_fields.items() if not value]
|
|
74
|
+
if missing_fields:
|
|
75
|
+
raise ValueError(f"Missing required configuration: {', '.join(missing_fields)}")
|
|
76
|
+
|
|
77
|
+
self._log_ingestion_endpoint = f"{self._sentinel_dce_log_ingestion_url}/dataCollectionRules/{self._sentinel_dce_dcr_immutable_id}/streams/{self._sentinel_dce_dcr_stream_id}?api-version=2023-01-01"
|
|
78
|
+
self._batch_size = cfg.get("batch_size", self.DEFAULT_BATCH_SIZE)
|
|
79
|
+
|
|
80
|
+
def process_lines(self, lines: Iterable[str]) -> None:
|
|
81
|
+
"""
|
|
82
|
+
Process log lines and send them in batches to Azure Monitor.
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
lines: Iterable of log line strings (should be valid JSON)
|
|
86
|
+
"""
|
|
87
|
+
processed_lines = 0
|
|
88
|
+
batch_num = 0
|
|
89
|
+
batch_items = []
|
|
90
|
+
|
|
91
|
+
for line in lines:
|
|
92
|
+
batch_items.append(line)
|
|
93
|
+
processed_lines += 1
|
|
94
|
+
|
|
95
|
+
if processed_lines % self._batch_size == 0:
|
|
96
|
+
self.log.info(
|
|
97
|
+
"[sentinel, SendBatchToMonitor] Processed %d lines. Sending batch %d to monitor...",
|
|
98
|
+
processed_lines, batch_num
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
self._stream_to_monitor(batch_items)
|
|
102
|
+
metrics.inc("sentinel_batch_sent")
|
|
103
|
+
|
|
104
|
+
batch_num += 1
|
|
105
|
+
batch_items = []
|
|
106
|
+
|
|
107
|
+
# Send remaining items in final batch
|
|
108
|
+
if batch_items:
|
|
109
|
+
self.log.info(
|
|
110
|
+
"[sentinel, SendBatchToMonitor] Processed %d lines. Sending final batch %d to monitor...",
|
|
111
|
+
processed_lines, batch_num
|
|
112
|
+
)
|
|
113
|
+
self._stream_to_monitor(batch_items)
|
|
114
|
+
metrics.inc("sentinel_batch_sent")
|
|
115
|
+
batch_num += 1
|
|
116
|
+
|
|
117
|
+
self.log.info(
|
|
118
|
+
"[sentinel, BlobProcessingCompleted] Finished processing. Total processed lines: %d, Total batches sent: %d.",
|
|
119
|
+
processed_lines, batch_num
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
def _stream_to_monitor(self, batch_items: list) -> None:
|
|
123
|
+
"""
|
|
124
|
+
Send a batch of log items to Azure Monitor.
|
|
125
|
+
|
|
126
|
+
Args:
|
|
127
|
+
batch_items: List of log line strings to send
|
|
128
|
+
"""
|
|
129
|
+
try:
|
|
130
|
+
access_token = self._get_monitor_token()
|
|
131
|
+
|
|
132
|
+
headers = {
|
|
133
|
+
"Authorization": f"Bearer {access_token}",
|
|
134
|
+
"Content-Type": "application/json"
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
# Build JSON array payload
|
|
138
|
+
payload = "[" + ", ".join(batch_items) + "]"
|
|
139
|
+
|
|
140
|
+
self.log.info(
|
|
141
|
+
"[sentinel, SendingToAzureMonitor] Sending payload to %s...",
|
|
142
|
+
self._log_ingestion_endpoint
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
response = requests.post(
|
|
146
|
+
self._log_ingestion_endpoint,
|
|
147
|
+
data=payload,
|
|
148
|
+
headers=headers,
|
|
149
|
+
timeout=30
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
self.log.info(
|
|
153
|
+
"[sentinel, SendingToAzureMonitor] Result code is %s...",
|
|
154
|
+
response.status_code
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
response.raise_for_status()
|
|
158
|
+
metrics.inc("sentinel_send_success")
|
|
159
|
+
|
|
160
|
+
except requests.exceptions.RequestException as e:
|
|
161
|
+
self.log.error("Failed to send to Azure Monitor: %s", str(e))
|
|
162
|
+
metrics.inc("sentinel_send_error")
|
|
163
|
+
raise
|
|
164
|
+
except Exception as e:
|
|
165
|
+
self.log.error("Unexpected error sending to Azure Monitor: %s", str(e))
|
|
166
|
+
metrics.inc("sentinel_send_error")
|
|
167
|
+
raise
|
|
168
|
+
|
|
169
|
+
def _get_monitor_token(self) -> str:
|
|
170
|
+
"""
|
|
171
|
+
Get or refresh the Azure Monitor access token.
|
|
172
|
+
|
|
173
|
+
Returns:
|
|
174
|
+
Valid access token string
|
|
175
|
+
"""
|
|
176
|
+
now = datetime.now(timezone.utc)
|
|
177
|
+
token_expiry_buffer = timedelta(minutes=self.TOKEN_REFRESH_BUFFER_MINUTES)
|
|
178
|
+
expiry_threshold = (now + token_expiry_buffer).timestamp()
|
|
179
|
+
|
|
180
|
+
# Check if we need to refresh the token
|
|
181
|
+
if (SentinelProcessor._monitor_token is None or
|
|
182
|
+
SentinelProcessor._monitor_token.expires_on < expiry_threshold):
|
|
183
|
+
|
|
184
|
+
credential = self._get_credential()
|
|
185
|
+
|
|
186
|
+
self.log.debug("[sentinel, AzureMonitorCredentials] Requesting new token from Azure...")
|
|
187
|
+
SentinelProcessor._monitor_token = credential.get_token(self.MONITOR_SCOPE)
|
|
188
|
+
self.log.info("[sentinel, AzureMonitorCredentials] Token acquired, expires at %s",
|
|
189
|
+
SentinelProcessor._monitor_token.expires_on)
|
|
190
|
+
else:
|
|
191
|
+
self.log.info("[sentinel, AzureMonitorCredentials] Using cached token for Azure Monitor...")
|
|
192
|
+
|
|
193
|
+
return SentinelProcessor._monitor_token.token
|
|
194
|
+
|
|
195
|
+
def _get_credential(self):
|
|
196
|
+
"""
|
|
197
|
+
Get the appropriate Azure credential based on configuration.
|
|
198
|
+
|
|
199
|
+
Returns:
|
|
200
|
+
Azure credential object (ClientSecretCredential or DefaultAzureCredential)
|
|
201
|
+
"""
|
|
202
|
+
if self._sentinel_dce_application_secret and self._sentinel_dce_application_id and self._sentinel_dce_tenant_id:
|
|
203
|
+
self.log.info(
|
|
204
|
+
"[sentinel, AzureMonitorCredentials] Using ClientSecretCredential with Tenant ID %s, Client ID %s to get new token for Azure Monitor...",
|
|
205
|
+
self._sentinel_dce_tenant_id, self._sentinel_dce_application_id
|
|
206
|
+
)
|
|
207
|
+
return ClientSecretCredential(
|
|
208
|
+
tenant_id=decrypt_value_if_needed(self._sentinel_dce_tenant_id),
|
|
209
|
+
client_id=decrypt_value_if_needed(self._sentinel_dce_application_id),
|
|
210
|
+
client_secret=decrypt_value_if_needed(self._sentinel_dce_application_secret)
|
|
211
|
+
)
|
|
212
|
+
else:
|
|
213
|
+
self.log.info(
|
|
214
|
+
"[sentinel, AzureMonitorCredentials] Using DefaultAzureCredential to get new token for Azure Monitor..."
|
|
215
|
+
)
|
|
216
|
+
return DefaultAzureCredential()
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def send_sentinel_dce(lines: Iterable[str], cfg: dict) -> None:
|
|
220
|
+
"""
|
|
221
|
+
Output handler function for sending logs to Azure Sentinel.
|
|
222
|
+
|
|
223
|
+
This function is designed to be used as an output handler in the processor module.
|
|
224
|
+
|
|
225
|
+
Args:
|
|
226
|
+
lines: Iterable of log line strings
|
|
227
|
+
cfg: Configuration dictionary with sentinel-specific settings
|
|
228
|
+
"""
|
|
229
|
+
processor = SentinelProcessor(cfg)
|
|
230
|
+
processor.process_lines(lines)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: sap-ecs-log-forwarder
|
|
3
|
-
Version: 1.0
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: A package to consume events from an AWS SQS queue, Azure Storage Account Topic and GCP PubSub Topic, process log files, and forward them to a HTTP endpoint or file.
|
|
5
5
|
Home-page: https://www.sap.com/
|
|
6
6
|
License: SAP DEVELOPER LICENSE AGREEMENT
|
|
@@ -20,6 +20,7 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.14
|
|
21
21
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
22
|
Requires-Dist: aiohttp (>=3.13.2,<4.0.0)
|
|
23
|
+
Requires-Dist: azure-identity (>=1.25.1,<2.0.0)
|
|
23
24
|
Requires-Dist: azure-storage-blob (>=12.27.1,<13.0.0)
|
|
24
25
|
Requires-Dist: azure-storage-queue (>=12.14.1,<13.0.0)
|
|
25
26
|
Requires-Dist: boto3 (>=1.17.49,<2.0.0)
|
|
@@ -38,13 +39,13 @@ Unified Python service that consumes object / blob creation events from:
|
|
|
38
39
|
- GCP Pub/Sub (Cloud Storage OBJECT_FINALIZE)
|
|
39
40
|
- Azure Queue (Storage BlobCreated events)
|
|
40
41
|
|
|
41
|
-
Downloads referenced log files, decompresses if gzip, splits into lines, and forwards each line to configured outputs (HTTP, files, console). Provides structured JSON logging, in‑memory metrics, regex filtering, encrypted credentials, jittered retries, and a configuration CLI.
|
|
42
|
+
Downloads referenced log files, decompresses if gzip, splits into lines, and forwards each line to configured outputs (HTTP, files, console, sentinel). Provides structured JSON logging, in‑memory metrics, regex filtering, encrypted credentials, jittered retries, and a configuration CLI.
|
|
42
43
|
|
|
43
44
|
## Features
|
|
44
45
|
- Single `config.json` drives all cloud provider inputs.
|
|
45
46
|
- Per input regex filters: include / exclude.
|
|
46
47
|
- Output‑level filters (include / exclude) per output.
|
|
47
|
-
- Outputs: http, files, console (multiple per input).
|
|
48
|
+
- Outputs: http, files, console, sentinel (multiple per input).
|
|
48
49
|
- HTTP output supports TLS client certs, custom CA, and insecure skip verify.
|
|
49
50
|
- Encrypted credentials stored inline (Fernet) using `enc:` prefix.
|
|
50
51
|
- Structured JSON logging (console by default, optional file target).
|
|
@@ -128,6 +129,35 @@ Provider authentication fields:
|
|
|
128
129
|
- GCP: serviceAccountJson (raw JSON string encrypted)
|
|
129
130
|
All optionally encrypted with `enc:` prefix.
|
|
130
131
|
|
|
132
|
+
### Sending logs to Sentinel
|
|
133
|
+
|
|
134
|
+
To send logs to an Azure Log Analytics Workspace for further analysis with Sentinel, set up the "SAP LogServ (RISE), S/4 HANA Cloud Private Edition" from the Microsoft Sentinel Content Hub as described [here](https://community.sap.com/t5/enterprise-resource-planning-blog-posts-by-sap/sap-logserv-integration-with-microsoft-sentinel-for-sap-rise-customers-is/ba-p/14085387). The deployment will provide the values for all `sentinel_...`-prefixed properties shown in the code snippet to define the `sentinel` output type below.
|
|
135
|
+
|
|
136
|
+
```
|
|
137
|
+
{
|
|
138
|
+
...
|
|
139
|
+
"inputs": [
|
|
140
|
+
{
|
|
141
|
+
...
|
|
142
|
+
"outputs": [
|
|
143
|
+
{
|
|
144
|
+
"type": "sentinel",
|
|
145
|
+
"includeFilter": [],
|
|
146
|
+
"excludeFilter": [],
|
|
147
|
+
"sentinel_dce_tenant_id": "<Destination tenant ID>",
|
|
148
|
+
"sentinel_dce_application_id": "<Destination Application / client ID>",
|
|
149
|
+
"sentinel_dce_application_secret": "<Destination client secret>",
|
|
150
|
+
"sentinel_dce_log_ingestion_url": "<Log ingestion base URL>",
|
|
151
|
+
"sentinel_dce_dcr_immutable_id": "<Data Collection Rule immutable ID>",
|
|
152
|
+
"sentinel_dce_dcr_stream_id": "Custom-SAPLogServ_CL"
|
|
153
|
+
}
|
|
154
|
+
]
|
|
155
|
+
}
|
|
156
|
+
]
|
|
157
|
+
}
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
|
|
131
161
|
## Installation
|
|
132
162
|
### With direct internet access:
|
|
133
163
|
```sh
|
|
@@ -175,6 +205,11 @@ List inputs:
|
|
|
175
205
|
sap-ecs-config-cli input list
|
|
176
206
|
```
|
|
177
207
|
|
|
208
|
+
Show input:
|
|
209
|
+
```bash
|
|
210
|
+
sap-ecs-config-cli input show aws1
|
|
211
|
+
```
|
|
212
|
+
|
|
178
213
|
Remove input:
|
|
179
214
|
```bash
|
|
180
215
|
sap-ecs-config-cli input remove aws1
|
|
@@ -186,11 +221,27 @@ sap-ecs-config-cli output add --input-name aws1 --type http --destination https:
|
|
|
186
221
|
--include prod --exclude test
|
|
187
222
|
```
|
|
188
223
|
|
|
224
|
+
Add output for ingestion into Microsoft Sentinel via Data Collection Endpoint (DCE):
|
|
225
|
+
```bash
|
|
226
|
+
sap-ecs-config-cli output add --input-name azure1 \
|
|
227
|
+
--type sentinel \
|
|
228
|
+
--sentinel-dce-tenant-id "11111111-2222-3333-4444-555555555555" \
|
|
229
|
+
--sentinel-dce-app-id "11111111-2222-3333-4444-555555555555" \
|
|
230
|
+
--sentinel-dce-app-secret "the-secret" \
|
|
231
|
+
--sentinel-dce-ingestion-url "https://asi-....ingest.monitor.azure.com" \
|
|
232
|
+
--sentinel-dce-dcr-immutable-id "dcr-..."
|
|
233
|
+
```
|
|
234
|
+
|
|
189
235
|
List outputs:
|
|
190
236
|
```bash
|
|
191
237
|
sap-ecs-config-cli output list aws1
|
|
192
238
|
```
|
|
193
239
|
|
|
240
|
+
Show output:
|
|
241
|
+
```bash
|
|
242
|
+
sap-ecs-config-cli output show --input-name aws1 --index 0
|
|
243
|
+
```
|
|
244
|
+
|
|
194
245
|
Set HTTP authorization (encrypted):
|
|
195
246
|
```bash
|
|
196
247
|
sap-ecs-config-cli creds set-http-auth --input-name aws1 --output-index 0 --auth-type bearer
|
|
@@ -629,6 +680,10 @@ This application and its source code are licensed under the terms of the SAP Dev
|
|
|
629
680
|
|
|
630
681
|
## Changelog
|
|
631
682
|
|
|
683
|
+
**Version 1.1.0**
|
|
684
|
+
|
|
685
|
+
- Added output type `sentinel` to ingest data into a Microsoft Sentinel deployment through the solution "SAP LogServ (RISE), S/4 HANA Cloud Private Edition" from the Microsoft Sentinel Content Hub as described [here](https://community.sap.com/t5/enterprise-resource-planning-blog-posts-by-sap/sap-logserv-integration-with-microsoft-sentinel-for-sap-rise-customers-is/ba-p/14085387).
|
|
686
|
+
|
|
632
687
|
**Version 1.0.2**
|
|
633
688
|
|
|
634
689
|
- Solving minor issue to avoid data loss when HTTP destination is unavailable
|
|
@@ -2,17 +2,18 @@ sap_ecs_log_forwarder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
|
|
|
2
2
|
sap_ecs_log_forwarder/aws.py,sha256=1DXtV1aAbmciCxoU3T99nXPH0cXIbCu1DOV85j-RpsU,10149
|
|
3
3
|
sap_ecs_log_forwarder/azure.py,sha256=ZeOltwwZHQXVIZZ7TKzO4px5VGFqnH3kAYKWRwhP0Z0,12099
|
|
4
4
|
sap_ecs_log_forwarder/base_runner.py,sha256=PVCpKcUtRO5v-_7oD5kN6Ty-P5URwesU17_BjraHB3c,586
|
|
5
|
-
sap_ecs_log_forwarder/cli.py,sha256=
|
|
5
|
+
sap_ecs_log_forwarder/cli.py,sha256=53d8N9ckl_AxWJWV4eVCOUhNFsyFCIYUhjUzeP3oOFU,17582
|
|
6
6
|
sap_ecs_log_forwarder/config.py,sha256=_zr48_xDn2XEPwpNnQelaBBZXU5SVa6cJPAzNTaM9hk,3446
|
|
7
7
|
sap_ecs_log_forwarder/consumer.py,sha256=ODdlThC-yzi1A5MdZcrlaoWiIC9y9C2R7hEoTCxh2vI,1738
|
|
8
8
|
sap_ecs_log_forwarder/crypto.py,sha256=MCn2K3PLecRdmtW-6yvT-5hlLHWdVhNUH83OlqYNitA,1104
|
|
9
9
|
sap_ecs_log_forwarder/gcp.py,sha256=0MP0izDIBvUo7ZuGIDOQvrk9CxIe9zHdvDUhVDqNcTM,4449
|
|
10
10
|
sap_ecs_log_forwarder/json_logging.py,sha256=2kLmBfhq-qlj0Lmm1fPKz6qDLFJxV0Ga37JklrYFrL4,1308
|
|
11
11
|
sap_ecs_log_forwarder/metrics.py,sha256=XkEBsVA9RuLucEU_8TCdeNSrJXB3dNThGsgubVkyZiw,871
|
|
12
|
-
sap_ecs_log_forwarder/processor.py,sha256
|
|
12
|
+
sap_ecs_log_forwarder/processor.py,sha256=HAMMBLUAMhN_7CnuKeo3V9lcIsoybsOI_R48fdbnakY,4233
|
|
13
|
+
sap_ecs_log_forwarder/processor_sentinel.py,sha256=LV9c31Ku-JKfIS07FcJGOMAq8ww6LdGan_1yasNTS8M,9441
|
|
13
14
|
sap_ecs_log_forwarder/utils.py,sha256=mVKEEjUfWjQGBn4OsvIDCeNysjEkU1x3loJ1A8IMCKs,852
|
|
14
|
-
sap_ecs_log_forwarder-1.0.
|
|
15
|
-
sap_ecs_log_forwarder-1.0.
|
|
16
|
-
sap_ecs_log_forwarder-1.0.
|
|
17
|
-
sap_ecs_log_forwarder-1.0.
|
|
18
|
-
sap_ecs_log_forwarder-1.0.
|
|
15
|
+
sap_ecs_log_forwarder-1.1.0.dist-info/LICENSE,sha256=RTHTDJe35fXCEJkg3wT9DHAO8QQZWRNd9NELcyj4jN0,13528
|
|
16
|
+
sap_ecs_log_forwarder-1.1.0.dist-info/METADATA,sha256=-sCVQOMb9WcCsk7qyi0g8WBNsLvyzPi8xKmN7etNpNA,22612
|
|
17
|
+
sap_ecs_log_forwarder-1.1.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
18
|
+
sap_ecs_log_forwarder-1.1.0.dist-info/entry_points.txt,sha256=NHEVCnnKA3LQ-vynlLo2sf9hOGhbNrricddmwY8JQeA,129
|
|
19
|
+
sap_ecs_log_forwarder-1.1.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
{sap_ecs_log_forwarder-1.0.2.dist-info → sap_ecs_log_forwarder-1.1.0.dist-info}/entry_points.txt
RENAMED
|
File without changes
|