sap-ecs-log-forwarder 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sap_ecs_log_forwarder/__init__.py +0 -0
- sap_ecs_log_forwarder/aws.py +208 -0
- sap_ecs_log_forwarder/azure.py +240 -0
- sap_ecs_log_forwarder/base_runner.py +26 -0
- sap_ecs_log_forwarder/cli.py +303 -0
- sap_ecs_log_forwarder/config.py +99 -0
- sap_ecs_log_forwarder/consumer.py +56 -0
- sap_ecs_log_forwarder/crypto.py +41 -0
- sap_ecs_log_forwarder/gcp.py +105 -0
- sap_ecs_log_forwarder/json_logging.py +42 -0
- sap_ecs_log_forwarder/metrics.py +37 -0
- sap_ecs_log_forwarder/processor.py +107 -0
- sap_ecs_log_forwarder/utils.py +28 -0
- sap_ecs_log_forwarder-1.0.0.dist-info/LICENSE +38 -0
- sap_ecs_log_forwarder-1.0.0.dist-info/METADATA +634 -0
- sap_ecs_log_forwarder-1.0.0.dist-info/RECORD +18 -0
- sap_ecs_log_forwarder-1.0.0.dist-info/WHEEL +4 -0
- sap_ecs_log_forwarder-1.0.0.dist-info/entry_points.txt +4 -0
|
@@ -0,0 +1,634 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: sap-ecs-log-forwarder
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: A package to consume events from an AWS SQS queue, Azure Storage Account Topic and GCP PubSub Topic, process log files, and forward them to a HTTP endpoint or file.
|
|
5
|
+
Home-page: https://www.sap.com/
|
|
6
|
+
License: SAP DEVELOPER LICENSE AGREEMENT
|
|
7
|
+
Keywords: SAP AWS Log Forwarder,SAP Azure Log Forwarder,SAP GCP Log Forwarder,SAP Log Forwarder
|
|
8
|
+
Author: SAP SE
|
|
9
|
+
Requires-Python: >=3.10,<4.0
|
|
10
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: Other/Proprietary License
|
|
13
|
+
Classifier: Operating System :: MacOS :: MacOS X
|
|
14
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
|
+
Requires-Dist: aiohttp (>=3.13.2,<4.0.0)
|
|
23
|
+
Requires-Dist: azure-storage-blob (>=12.27.1,<13.0.0)
|
|
24
|
+
Requires-Dist: azure-storage-queue (>=12.14.1,<13.0.0)
|
|
25
|
+
Requires-Dist: boto3 (>=1.17.49,<2.0.0)
|
|
26
|
+
Requires-Dist: click (>=8.3.0,<9.0.0)
|
|
27
|
+
Requires-Dist: cryptography (>=44.0.1,<45.0.0)
|
|
28
|
+
Requires-Dist: google-cloud-pubsub (>=2.33.0,<3.0.0)
|
|
29
|
+
Requires-Dist: google-cloud-storage (>=3.6.0,<4.0.0)
|
|
30
|
+
Requires-Dist: requests (>=2.32.4,<3.0.0)
|
|
31
|
+
Requires-Dist: urllib3 (>=2.5.0,<3.0.0)
|
|
32
|
+
Description-Content-Type: text/markdown
|
|
33
|
+
|
|
34
|
+
# SAP ECS Log Forwarder
|
|
35
|
+
|
|
36
|
+
Unified Python service that consumes object / blob creation events from:
|
|
37
|
+
- AWS SQS (S3 ObjectCreated notifications)
|
|
38
|
+
- GCP Pub/Sub (Cloud Storage OBJECT_FINALIZE)
|
|
39
|
+
- Azure Queue (Storage BlobCreated events)
|
|
40
|
+
|
|
41
|
+
Downloads referenced log files, decompresses if gzip, splits into lines, and forwards each line to configured outputs (HTTP, files, console). Provides structured JSON logging, in‑memory metrics, regex filtering, encrypted credentials, jittered retries, and a configuration CLI.
|
|
42
|
+
|
|
43
|
+
## Features
|
|
44
|
+
- Single `config.json` drives all cloud provider inputs.
|
|
45
|
+
- Per input regex filters: include / exclude.
|
|
46
|
+
- Output‑level filters (include / exclude) per output.
|
|
47
|
+
- Outputs: http, files, console (multiple per input).
|
|
48
|
+
- HTTP output supports TLS client certs, custom CA, and insecure skip verify.
|
|
49
|
+
- Encrypted credentials stored inline (Fernet) using `enc:` prefix.
|
|
50
|
+
- Structured JSON logging (console by default, optional file target).
|
|
51
|
+
- In‑memory counters periodically logged; daily reset at 00:00 UTC.
|
|
52
|
+
- Jittered exponential retries for transient failures.
|
|
53
|
+
- CLI to manage inputs, outputs, credentials, and log file path.
|
|
54
|
+
- Unit test scaffolding (pytest).
|
|
55
|
+
|
|
56
|
+
## Architecture
|
|
57
|
+
|
|
58
|
+
### AWS
|
|
59
|
+
|
|
60
|
+

|
|
61
|
+
|
|
62
|
+
### Azure
|
|
63
|
+
|
|
64
|
+

|
|
65
|
+
|
|
66
|
+
### GCP
|
|
67
|
+
|
|
68
|
+

|
|
69
|
+
|
|
70
|
+
## Configuration Schema (config.json)
|
|
71
|
+
```json
|
|
72
|
+
{
|
|
73
|
+
"logLevel": "INFO",
|
|
74
|
+
"logFile": "/var/log/sap-log-forwarder/app.jsonl",
|
|
75
|
+
"inputs": [
|
|
76
|
+
{
|
|
77
|
+
"provider": "aws",
|
|
78
|
+
"name": "aws1",
|
|
79
|
+
"queue": "https://sqs.us-east-1.amazonaws.com/123/queue",
|
|
80
|
+
"region": "us-east-1",
|
|
81
|
+
"bucket": "my-bucket",
|
|
82
|
+
"includeFilter": ["\\.log$"],
|
|
83
|
+
"excludeFilter": ["debug"],
|
|
84
|
+
"maxRetries": 5,
|
|
85
|
+
"retryDelay": 10,
|
|
86
|
+
"authentication": {
|
|
87
|
+
"accessKeyId": "enc:...",
|
|
88
|
+
"secretAccessKey": "enc:...",
|
|
89
|
+
"encrypted": true
|
|
90
|
+
},
|
|
91
|
+
"outputs": [
|
|
92
|
+
{
|
|
93
|
+
"type": "http",
|
|
94
|
+
"destination": "https://example.com/ingest",
|
|
95
|
+
"authorization": {
|
|
96
|
+
"type": "bearer",
|
|
97
|
+
"token": "enc:...",
|
|
98
|
+
"encrypted": true
|
|
99
|
+
},
|
|
100
|
+
"tls": {
|
|
101
|
+
"pathToClientCert": "/path/client.crt",
|
|
102
|
+
"pathToClientKey": "/path/client.key",
|
|
103
|
+
"pathToCACert": "/path/ca.crt",
|
|
104
|
+
"insecureSkipVerify": false
|
|
105
|
+
},
|
|
106
|
+
"includeFilter": ["prod"],
|
|
107
|
+
"excludeFilter": ["test"]
|
|
108
|
+
},
|
|
109
|
+
{
|
|
110
|
+
"type": "files",
|
|
111
|
+
"destination": "logs/",
|
|
112
|
+
"compress": true,
|
|
113
|
+
"includeFilter": [],
|
|
114
|
+
"excludeFilter": []
|
|
115
|
+
},
|
|
116
|
+
{
|
|
117
|
+
"type": "console"
|
|
118
|
+
}
|
|
119
|
+
]
|
|
120
|
+
}
|
|
121
|
+
]
|
|
122
|
+
}
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Provider authentication fields:
|
|
126
|
+
- AWS: accessKeyId, secretAccessKey; or dynamic clientId, clientSecret, loginUrl, awsCredsUrl
|
|
127
|
+
- Azure: sasToken; or dynamic clientId, clientSecret, loginUrl, credsUrl
|
|
128
|
+
- GCP: serviceAccountJson (raw JSON string encrypted)
|
|
129
|
+
All optionally encrypted with `enc:` prefix.
|
|
130
|
+
|
|
131
|
+
## Installation
|
|
132
|
+
### With direct internet access:
|
|
133
|
+
```sh
|
|
134
|
+
pip install sap-ecs-log-forwarder
|
|
135
|
+
|
|
136
|
+
or
|
|
137
|
+
|
|
138
|
+
pip install sap-ecs-log-forwarder==<version>
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
### Without direct internet access:
|
|
142
|
+
1. **Download the wheel**
|
|
143
|
+
On a machine with internet access, visit the [PyPI files page](https://pypi.org/project/sap-ecs-log-forwarder/#files) for `sap-ecs-aws-log-forwarder` and download the `.whl` file matching your Python version (e.g., `sap_ecs_log_forwarder-1.0.0-py3-none-any.whl`).
|
|
144
|
+
2. **Transfer the file**
|
|
145
|
+
Copy the downloaded wheel to the target (offline) machine using SCP, or another secure method.
|
|
146
|
+
3. **Install from file**
|
|
147
|
+
On the offline machine, run:
|
|
148
|
+
```sh
|
|
149
|
+
pip install /path/to/sap_ecs_log_forwarder-<version>-py3-none-any.whl
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
## Encryption
|
|
153
|
+
Generate a Fernet key:
|
|
154
|
+
```python
|
|
155
|
+
from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())
|
|
156
|
+
```
|
|
157
|
+
Export:
|
|
158
|
+
```bash
|
|
159
|
+
export FORWARDER_ENCRYPTION_KEY="YOUR_FERNET_KEY"
|
|
160
|
+
```
|
|
161
|
+
CLI will encrypt secrets you enter; encrypted values stored as `enc:<ciphertext>`.
|
|
162
|
+
|
|
163
|
+
## CLI Usage
|
|
164
|
+
Add an input:
|
|
165
|
+
```bash
|
|
166
|
+
sap-ecs-config-cli input add --provider aws --name aws1 --queue https://sqs... --region us-east-1 --bucket my-bucket
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
```bash
|
|
170
|
+
sap-ecs-config-cli input add --provider azure --name azure1 --queue my-queue --storage-account my-storage-account
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
List inputs:
|
|
174
|
+
```bash
|
|
175
|
+
sap-ecs-config-cli input list
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
Remove input:
|
|
179
|
+
```bash
|
|
180
|
+
sap-ecs-config-cli input remove aws1
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
Add output (with optional output-level filters):
|
|
184
|
+
```bash
|
|
185
|
+
sap-ecs-config-cli output add --input-name aws1 --type http --destination https://example.com/ingest \
|
|
186
|
+
--include prod --exclude test
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
List outputs:
|
|
190
|
+
```bash
|
|
191
|
+
sap-ecs-config-cli output list aws1
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
Set HTTP authorization (encrypted):
|
|
195
|
+
```bash
|
|
196
|
+
sap-ecs-config-cli creds set-http-auth --input-name aws1 --output-index 0 --auth-type bearer
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
Set provider credentials (encrypted):
|
|
200
|
+
```bash
|
|
201
|
+
sap-ecs-config-cli creds set-provider-auth --input-name aws1
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
Configure JSON log file path:
|
|
205
|
+
```bash
|
|
206
|
+
sap-ecs-config-cli set-log-file --path /var/log/sap-log-forwarder/app.jsonl
|
|
207
|
+
# To disable file logging:
|
|
208
|
+
sap-ecs-config-cli set-log-file --path ""
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
## Configuration File Location
|
|
212
|
+
|
|
213
|
+
Resolution order:
|
|
214
|
+
1. Environment variable `SAP_LOG_FORWARDER_CONFIG` if set (supports `~` expansion).
|
|
215
|
+
2. `./config.json` in current working directory (legacy behavior) if it exists.
|
|
216
|
+
3. Default: `~/.sapecslogforwarder/config.json` (directory auto-created).
|
|
217
|
+
|
|
218
|
+
Show path:
|
|
219
|
+
```bash
|
|
220
|
+
sap-ecs-config-cli config-path
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
Set custom path:
|
|
224
|
+
```bash
|
|
225
|
+
export SAP_LOG_FORWARDER_CONFIG=/etc/sap-log-forwarder/config.json
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
All CLI operations read/write the resolved path.
|
|
229
|
+
|
|
230
|
+
## Usage
|
|
231
|
+
|
|
232
|
+
To run the Log Forwarder, use the following command:
|
|
233
|
+
|
|
234
|
+
```sh
|
|
235
|
+
sap-ecs-log-forwarder
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
Service starts threads / async loops per input, logs metrics snapshot every 30s.
|
|
239
|
+
|
|
240
|
+
## Structured Logging
|
|
241
|
+
Emits JSON lines to console (and to logFile if configured):
|
|
242
|
+
```json
|
|
243
|
+
{"ts":"2025-11-25T12:00:00","level":"INFO","message":"Wrote logs to logs/app.log.gz","thread":"aws-aws1"}
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
## Metrics (logged)
|
|
247
|
+
Counters (examples):
|
|
248
|
+
- files_forward_success / files_forward_error
|
|
249
|
+
- http_forward_success / http_forward_error
|
|
250
|
+
- output_invocations
|
|
251
|
+
- aws_messages_processed / aws_retry
|
|
252
|
+
- gcp_messages_processed / gcp_retry
|
|
253
|
+
- azure_messages_processed / azure_retry
|
|
254
|
+
|
|
255
|
+
Snapshot logged periodically; can be extended to expose HTTP endpoint.
|
|
256
|
+
|
|
257
|
+
## Filtering
|
|
258
|
+
- Include / exclude lists are regex patterns applied (case-insensitive) to object/blob/file names.
|
|
259
|
+
- Exclude overrides include.
|
|
260
|
+
- Output-level include/exclude filters are also supported per output.
|
|
261
|
+
|
|
262
|
+
## Retries
|
|
263
|
+
Exponential backoff with added random jitter up to base retryDelay for download / processing failures.
|
|
264
|
+
|
|
265
|
+
## HTTP Output
|
|
266
|
+
- Per-line POST with `Content-Type: application/json`.
|
|
267
|
+
- Auth types: bearer, api-key, basic.
|
|
268
|
+
- TLS options:
|
|
269
|
+
- pathToClientCert + pathToClientKey for mTLS
|
|
270
|
+
- pathToCACert for custom CA
|
|
271
|
+
- insecureSkipVerify to skip verification (not recommended)
|
|
272
|
+
|
|
273
|
+
## Extending
|
|
274
|
+
- Add new output: implement handler and register in `processor.py` (OUTPUT_HANDLERS).
|
|
275
|
+
- Add metrics: `metrics.inc("<name>")`.
|
|
276
|
+
- Add provider: create runner class following existing pattern and register in `consumer.PROVIDERS`.
|
|
277
|
+
|
|
278
|
+
## Security Notes
|
|
279
|
+
- Ensure `FORWARDER_ENCRYPTION_KEY` is injected via secrets manager.
|
|
280
|
+
- Structured logs avoid sensitive fields.
|
|
281
|
+
|
|
282
|
+
### AWS Dynamic Authentication (Temporary Credentials)
|
|
283
|
+
Instead of static IAM keys, you can configure dynamic AWS auth. The forwarder will:
|
|
284
|
+
- POST to your backend login endpoint with `client_id` and `client_secret`, receive `session-id-raven` cookie.
|
|
285
|
+
- GET temporary AWS credentials for the target bucket, and refresh them every ~15 minutes or before expiration.
|
|
286
|
+
|
|
287
|
+
CLI:
|
|
288
|
+
```bash
|
|
289
|
+
sap-ecs-config-cli creds set-provider-auth --input-name aws1
|
|
290
|
+
# Choose auth mode: dynamic
|
|
291
|
+
# Provide client_id, client_secret, login URL (default http://logserv.forwarder.host/api/v1/app/login)
|
|
292
|
+
# Provide AWS credentials URL (default http://logserv.forwarder.host/api/v1/aws/credentials)
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
Config (snippet):
|
|
296
|
+
```json
|
|
297
|
+
{
|
|
298
|
+
"provider": "aws",
|
|
299
|
+
"name": "aws1",
|
|
300
|
+
"queue": "https://sqs.us-east-1.amazonaws.com/123/queue",
|
|
301
|
+
"region": "us-east-1",
|
|
302
|
+
"bucket": "my-s3-bucket",
|
|
303
|
+
"authentication": {
|
|
304
|
+
"clientId": "enc:...",
|
|
305
|
+
"clientSecret": "enc:...",
|
|
306
|
+
"loginUrl": "http://logserv.forwarder.host/api/v1/app/login",
|
|
307
|
+
"awsCredsUrl": "http://logserv.forwarder.host/api/v1/aws/credentials",
|
|
308
|
+
"encrypted": true
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
Backend contract:
|
|
314
|
+
```bash
|
|
315
|
+
# Login
|
|
316
|
+
curl -X POST -H 'content-type: application/x-www-form-urlencoded' \
|
|
317
|
+
-d client_id=xxx -d client_secret=xxx \
|
|
318
|
+
http://logserv.forwarder.host/api/v1/app/login
|
|
319
|
+
|
|
320
|
+
# Fetch temporary creds (cookie required)
|
|
321
|
+
curl -H 'Cookie: session-id-raven=xxxx' \
|
|
322
|
+
'http://logserv.forwarder.host/api/v1/aws/credentials?bucket=my-s3-bucket'
|
|
323
|
+
```
|
|
324
|
+
|
|
325
|
+
Expected response:
|
|
326
|
+
```json
|
|
327
|
+
{
|
|
328
|
+
"data": {
|
|
329
|
+
"AccessKeyId": "xxxx",
|
|
330
|
+
"SecretAccessKey": "xxxxx",
|
|
331
|
+
"SessionToken": "xxxxxx",
|
|
332
|
+
"Expiration": "2025-11-27T00:20:33Z",
|
|
333
|
+
"Region": "ap-south-1"
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
```
|
|
337
|
+
|
|
338
|
+
Notes:
|
|
339
|
+
- Region from response overrides input region if provided; otherwise uses configured `region`.
|
|
340
|
+
- Static keys are used if dynamic fields are absent.
|
|
341
|
+
- Credentials refresh occurs on a schedule or 60s before expiration.
|
|
342
|
+
|
|
343
|
+
### Azure Dynamic Authentication (Temporary SAS)
|
|
344
|
+
Instead of a static `sasToken`, you can configure dynamic Azure auth. The forwarder will:
|
|
345
|
+
- POST to your backend login endpoint with `client_id` and `client_secret`, receive `session-id-raven` cookie.
|
|
346
|
+
- GET temporary credentials for the storage account, extract `SASToken`, and refresh it every ~15 minutes (or before expiry).
|
|
347
|
+
|
|
348
|
+
CLI:
|
|
349
|
+
```bash
|
|
350
|
+
sap-ecs-config-cli creds set-provider-auth --input-name azure1
|
|
351
|
+
# Choose auth mode: dynamic
|
|
352
|
+
# Provide client_id, client_secret, login URL (default http://logserv.forwarder.host/api/v1/app/login)
|
|
353
|
+
# Provide credentials URL (default http://logserv.forwarder.host/api/v1/azure/credentials)
|
|
354
|
+
```
|
|
355
|
+
|
|
356
|
+
Config (snippet):
|
|
357
|
+
```json
|
|
358
|
+
{
|
|
359
|
+
"provider": "azure",
|
|
360
|
+
"name": "azure1",
|
|
361
|
+
"queue": "my-queue",
|
|
362
|
+
"storageAccount": "mystorageacct",
|
|
363
|
+
"authentication": {
|
|
364
|
+
"clientId": "enc:...",
|
|
365
|
+
"clientSecret": "enc:...",
|
|
366
|
+
"loginUrl": "http://logserv.forwarder.host/api/v1/app/login",
|
|
367
|
+
"credsUrl": "http://logserv.forwarder.host/api/v1/azure/credentials",
|
|
368
|
+
"encrypted": true
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
```
|
|
372
|
+
|
|
373
|
+
Notes:
|
|
374
|
+
- The credentials endpoint must accept `storage-account-name` query parameter.
|
|
375
|
+
- Response must include `data.SASToken` and `data.Expiration` (RFC3339 UTC).
|
|
376
|
+
- If `sasToken` is set and dynamic fields absent, static SAS is used.
|
|
377
|
+
- For connection strings, the runner uses them directly and skips SAS refresh.
|
|
378
|
+
|
|
379
|
+
```bash
|
|
380
|
+
# macOS: test endpoints locally
|
|
381
|
+
curl -X POST -H "content-type: application/x-www-form-urlencoded" \
|
|
382
|
+
-d "client_id=xxx" -d "client_secret=xxx" \
|
|
383
|
+
http://logserv.forwarder.host/api/v1/app/login
|
|
384
|
+
|
|
385
|
+
curl -H "Cookie: session-id-raven=xxxxx" \
|
|
386
|
+
"http://logserv.forwarder.host/api/v1/azure/credentials?storage-account-name=mystorageacct"
|
|
387
|
+
```
|
|
388
|
+
|
|
389
|
+
### GCP Dynamic Authentication (Temporary Credentials)
|
|
390
|
+
|
|
391
|
+
**Will be available in a future release**
|
|
392
|
+
|
|
393
|
+
## Systemd Deployment Guide
|
|
394
|
+
(For Linux servers only.)
|
|
395
|
+
|
|
396
|
+
### Overview
|
|
397
|
+
Run the forwarder as a managed service with automatic restarts and persistent log files.
|
|
398
|
+
|
|
399
|
+
### 1. Create a Service Account (Optional)
|
|
400
|
+
```bash
|
|
401
|
+
sudo useradd -r -s /bin/false saplogfwd || true
|
|
402
|
+
```
|
|
403
|
+
|
|
404
|
+
### 2. Create Directories
|
|
405
|
+
```bash
|
|
406
|
+
sudo mkdir -p /opt/sap-log-forwarder
|
|
407
|
+
sudo mkdir -p /var/log/sap-log-forwarder
|
|
408
|
+
sudo chown -R saplogfwd:saplogfwd /opt/sap-log-forwarder /var/log/sap-log-forwarder
|
|
409
|
+
```
|
|
410
|
+
|
|
411
|
+
Place your project (editable install or wheel) under /opt/sap-log-forwarder or install via pip:
|
|
412
|
+
```bash
|
|
413
|
+
cd /opt/sap-log-forwarder
|
|
414
|
+
sudo pip install sap-ecs-log-forwarder
|
|
415
|
+
```
|
|
416
|
+
|
|
417
|
+
### 3. Environment File
|
|
418
|
+
Create /etc/sap-log-forwarder.env:
|
|
419
|
+
```bash
|
|
420
|
+
SAP_LOG_FORWARDER_CONFIG=/etc/sap-log-forwarder/config.json
|
|
421
|
+
FORWARDER_ENCRYPTION_KEY=YOUR_FERNET_KEY
|
|
422
|
+
PYTHONUNBUFFERED=1
|
|
423
|
+
LOG_LEVEL=INFO
|
|
424
|
+
```
|
|
425
|
+
|
|
426
|
+
(Generate key earlier and replace.)
|
|
427
|
+
|
|
428
|
+
### 4. Systemd Unit
|
|
429
|
+
Create /etc/systemd/system/sap-log-forwarder.service:
|
|
430
|
+
```ini
|
|
431
|
+
[Unit]
|
|
432
|
+
Description=SAP ECS Log Forwarder
|
|
433
|
+
After=network.target
|
|
434
|
+
Wants=network-online.target
|
|
435
|
+
|
|
436
|
+
[Service]
|
|
437
|
+
Type=simple
|
|
438
|
+
User=saplogfwd
|
|
439
|
+
Group=saplogfwd
|
|
440
|
+
WorkingDirectory=/opt/sap-log-forwarder
|
|
441
|
+
EnvironmentFile=/etc/sap-log-forwarder.env
|
|
442
|
+
ExecStart=/usr/bin/python -m sap_ecs_log_forwarder.consumer
|
|
443
|
+
Restart=always
|
|
444
|
+
RestartSec=5
|
|
445
|
+
# File logging (append). Systemd 240+ supports append:
|
|
446
|
+
StandardOutput=append:/var/log/sap-log-forwarder/forwarder.out.log
|
|
447
|
+
StandardError=append:/var/log/sap-log-forwarder/forwarder.err.log
|
|
448
|
+
# Alternatively (recommended) use journald and remove StandardOutput/StandardError lines.
|
|
449
|
+
|
|
450
|
+
# Resource limits (optional)
|
|
451
|
+
NoNewPrivileges=yes
|
|
452
|
+
ProtectSystem=full
|
|
453
|
+
ProtectHome=yes
|
|
454
|
+
|
|
455
|
+
[Install]
|
|
456
|
+
WantedBy=multi-user.target
|
|
457
|
+
```
|
|
458
|
+
|
|
459
|
+
Reload and start:
|
|
460
|
+
```bash
|
|
461
|
+
sudo systemctl daemon-reload
|
|
462
|
+
sudo systemctl enable --now sap-log-forwarder
|
|
463
|
+
```
|
|
464
|
+
|
|
465
|
+
Status:
|
|
466
|
+
```bash
|
|
467
|
+
systemctl status sap-log-forwarder
|
|
468
|
+
```
|
|
469
|
+
|
|
470
|
+
### 5. Log Rotation (If Using File Output)
|
|
471
|
+
Create /etc/logrotate.d/sap-log-forwarder:
|
|
472
|
+
```conf
|
|
473
|
+
/var/log/sap-log-forwarder/forwarder.*.log {
|
|
474
|
+
daily
|
|
475
|
+
rotate 14
|
|
476
|
+
compress
|
|
477
|
+
missingok
|
|
478
|
+
notifempty
|
|
479
|
+
create 0640 saplogfwd saplogfwd
|
|
480
|
+
sharedscripts
|
|
481
|
+
postrotate
|
|
482
|
+
systemctl kill -s SIGUSR1 sap-log-forwarder || true
|
|
483
|
+
endscript
|
|
484
|
+
}
|
|
485
|
+
```
|
|
486
|
+
|
|
487
|
+
(Forwarder ignores SIGUSR1 now; postrotate just optional. Compression handled by logrotate.)
|
|
488
|
+
|
|
489
|
+
### 6. Using Journald Instead
|
|
490
|
+
Remove StandardOutput/StandardError lines to keep logs in journal:
|
|
491
|
+
```bash
|
|
492
|
+
journalctl -u sap-log-forwarder -f
|
|
493
|
+
```
|
|
494
|
+
To export periodically, use:
|
|
495
|
+
```bash
|
|
496
|
+
journalctl -u sap-log-forwarder --since "1 hour ago" > /var/log/sap-log-forwarder/hour.log
|
|
497
|
+
```
|
|
498
|
+
|
|
499
|
+
### 7. Config File Location
|
|
500
|
+
Place config.json in WorkingDirectory (/opt/sap-log-forwarder/config.json). Edit via CLI:
|
|
501
|
+
```bash
|
|
502
|
+
sudo -u saplogfwd sap-ecs-config-cli input list
|
|
503
|
+
```
|
|
504
|
+
|
|
505
|
+
### 8. Permissions
|
|
506
|
+
Ensure /var/log/sap-log-forwarder writable by service user. If using file outputs (destination logs/):
|
|
507
|
+
```bash
|
|
508
|
+
sudo mkdir -p /opt/sap-log-forwarder/logs
|
|
509
|
+
sudo chown saplogfwd:saplogfwd /opt/sap-log-forwarder/logs
|
|
510
|
+
```
|
|
511
|
+
|
|
512
|
+
### 9. Updating
|
|
513
|
+
```bash
|
|
514
|
+
sudo systemctl stop sap-log-forwarder
|
|
515
|
+
sudo pip install --upgrade sap-ecs-log-forwarder
|
|
516
|
+
sudo systemctl start sap-log-forwarder
|
|
517
|
+
```
|
|
518
|
+
|
|
519
|
+
### 10. Common Checks
|
|
520
|
+
```bash
|
|
521
|
+
systemctl status sap-log-forwarder
|
|
522
|
+
journalctl -u sap-log-forwarder -n 50
|
|
523
|
+
ls -lh /var/log/sap-log-forwarder
|
|
524
|
+
```
|
|
525
|
+
|
|
526
|
+
### 11. Troubleshooting Restart Loops
|
|
527
|
+
- Inspect recent logs (`journalctl -u sap-log-forwarder -f`).
|
|
528
|
+
- Validate config.json structure.
|
|
529
|
+
- Verify FORWARDER_ENCRYPTION_KEY present (`systemctl show -p Environment sap-log-forwarder`).
|
|
530
|
+
|
|
531
|
+
### 12. Security Hardening (Optional)
|
|
532
|
+
Add directives:
|
|
533
|
+
```
|
|
534
|
+
PrivateTmp=yes
|
|
535
|
+
ProtectKernelTunables=yes
|
|
536
|
+
ProtectKernelModules=yes
|
|
537
|
+
RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
|
|
538
|
+
```
|
|
539
|
+
|
|
540
|
+
Keep encryption key in a root-readable-only environment file (chmod 640).
|
|
541
|
+
|
|
542
|
+
### 13. Switching To Virtualenv
|
|
543
|
+
If installed in virtualenv:
|
|
544
|
+
```
|
|
545
|
+
ExecStart=/opt/sap-log-forwarder/venv/bin/python -m sap_ecs_log_forwarder.consumer
|
|
546
|
+
```
|
|
547
|
+
|
|
548
|
+
Ensure venv ownership matches service user.
|
|
549
|
+
|
|
550
|
+
### 14. Testing Manual Run
|
|
551
|
+
```bash
|
|
552
|
+
sudo -u saplogfwd FORWARDER_ENCRYPTION_KEY=YOUR_FERNET_KEY python -m sap_ecs_log_forwarder.consumer
|
|
553
|
+
```
|
|
554
|
+
|
|
555
|
+
Stop with:
|
|
556
|
+
```bash
|
|
557
|
+
sudo systemctl stop sap-log-forwarder
|
|
558
|
+
```
|
|
559
|
+
|
|
560
|
+
Service now runs with automatic restart (Restart=always) and logs persisted under /var/log/sap-log-forwarder.
|
|
561
|
+
|
|
562
|
+
## Troubleshooting
|
|
563
|
+
|
|
564
|
+
### Quick Checklist
|
|
565
|
+
1. config.json present and valid (`jq '.' config.json`).
|
|
566
|
+
2. `FORWARDER_ENCRYPTION_KEY` exported.
|
|
567
|
+
3. Inputs listed (`sap-ecs-config-cli input list`).
|
|
568
|
+
4. Outputs configured (`sap-ecs-config-cli output list <input>`).
|
|
569
|
+
5. Per-input `logLevel` set (use `DEBUG` for diagnosis).
|
|
570
|
+
6. If using HTTP/TLS, verify cert/key/CA paths.
|
|
571
|
+
|
|
572
|
+
### General Issues
|
|
573
|
+
| Symptom | Cause | Fix |
|
|
574
|
+
|---------|-------|-----|
|
|
575
|
+
| Encrypted values not decrypted | Missing `FORWARDER_ENCRYPTION_KEY` | Export key before running consumer |
|
|
576
|
+
| No files processed | Check exclude/include filters | Adjust filters |
|
|
577
|
+
| Output skipped | Output-level include/exclude filters | Remove or correct filters |
|
|
578
|
+
| HTTP send fails | Bad URL, TLS paths, undecrypted auth | Verify destination, cert/key/CA paths, export key |
|
|
579
|
+
| File output missing | No destination or permission denied | Provide writable directory in output config |
|
|
580
|
+
|
|
581
|
+
### AWS (SQS + S3)
|
|
582
|
+
- No messages: Ensure S3 event notification configured (ObjectCreated -> SQS).
|
|
583
|
+
- Objects ignored: Bucket mismatch, eventName not `ObjectCreated:Put`, regex filters.
|
|
584
|
+
- Repeated retries (`aws_retry`): Missing `s3:GetObject` or corrupt gzip.
|
|
585
|
+
- Credential failure: Re-run `creds set-provider-auth` and export key.
|
|
586
|
+
|
|
587
|
+
### GCP (Pub/Sub + Storage)
|
|
588
|
+
- Subscription path invalid: Must be `projects/<project>/subscriptions/<name>`.
|
|
589
|
+
- PermissionDenied: Grant `roles/pubsub.subscriber` and `roles/storage.objectViewer`.
|
|
590
|
+
- Service account JSON parse failure: Re-enter credentials (file/paste mode).
|
|
591
|
+
- Events ignored: Not `OBJECT_FINALIZE` or filters reject.
|
|
592
|
+
|
|
593
|
+
### Azure (Queue + Blob)
|
|
594
|
+
- Decode errors: Ensure Event Grid -> Queue message schema.
|
|
595
|
+
- Missing blob URL: Recreate Event Grid subscription; `data.url` must exist.
|
|
596
|
+
- Unauthorized blob download: Provide SAS token or full connection string.
|
|
597
|
+
- Filters miss: Subject pattern `/blobServices/default/containers/<c>/blobs/<name>`; adjust regex.
|
|
598
|
+
|
|
599
|
+
### Performance
|
|
600
|
+
- Slow: Large object fully loaded; consider future streaming enhancement.
|
|
601
|
+
- Many retries: Increase `retryDelay`, verify network and permissions.
|
|
602
|
+
- Parallelism: Each input runs its own thread/loop; split workloads across inputs.
|
|
603
|
+
|
|
604
|
+
### Debug Commands (macOS)
|
|
605
|
+
```bash
|
|
606
|
+
env | grep FORWARDER_ENCRYPTION_KEY
|
|
607
|
+
sap-ecs-config-cli input list
|
|
608
|
+
sap-ecs-config-cli output list <input-name>
|
|
609
|
+
python -m sap_ecs_log_forwarder.consumer
|
|
610
|
+
```
|
|
611
|
+
|
|
612
|
+
### Metrics Reference
|
|
613
|
+
- aws_messages_processed / aws_retry
|
|
614
|
+
- gcp_messages_processed / gcp_retry
|
|
615
|
+
- azure_messages_processed / azure_retry
|
|
616
|
+
- output_invocations
|
|
617
|
+
- files_forward_success / files_forward_error
|
|
618
|
+
- http_forward_success / http_forward_error
|
|
619
|
+
|
|
620
|
+
### When Nothing Processes
|
|
621
|
+
1. Set input `logLevel` to DEBUG.
|
|
622
|
+
2. Temporarily clear include/exclude filters.
|
|
623
|
+
3. Upload a small test file that matches expected pattern.
|
|
624
|
+
4. Confirm metrics counters increment.
|
|
625
|
+
5. Reapply filters gradually.
|
|
626
|
+
|
|
627
|
+
## License
|
|
628
|
+
This application and its source code are licensed under the terms of the SAP Developer License Agreement. See the LICENSE file for more information.
|
|
629
|
+
|
|
630
|
+
## Changelog
|
|
631
|
+
|
|
632
|
+
**Version 1.0.0**
|
|
633
|
+
|
|
634
|
+
- First proper release
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
sap_ecs_log_forwarder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
sap_ecs_log_forwarder/aws.py,sha256=1DXtV1aAbmciCxoU3T99nXPH0cXIbCu1DOV85j-RpsU,10149
|
|
3
|
+
sap_ecs_log_forwarder/azure.py,sha256=ZeOltwwZHQXVIZZ7TKzO4px5VGFqnH3kAYKWRwhP0Z0,12099
|
|
4
|
+
sap_ecs_log_forwarder/base_runner.py,sha256=PVCpKcUtRO5v-_7oD5kN6Ty-P5URwesU17_BjraHB3c,586
|
|
5
|
+
sap_ecs_log_forwarder/cli.py,sha256=MmPTP0UDtqlPQEYA0lRy_7qeTFne8dQIajxddhABYHQ,11718
|
|
6
|
+
sap_ecs_log_forwarder/config.py,sha256=_zr48_xDn2XEPwpNnQelaBBZXU5SVa6cJPAzNTaM9hk,3446
|
|
7
|
+
sap_ecs_log_forwarder/consumer.py,sha256=ODdlThC-yzi1A5MdZcrlaoWiIC9y9C2R7hEoTCxh2vI,1738
|
|
8
|
+
sap_ecs_log_forwarder/crypto.py,sha256=MCn2K3PLecRdmtW-6yvT-5hlLHWdVhNUH83OlqYNitA,1104
|
|
9
|
+
sap_ecs_log_forwarder/gcp.py,sha256=0MP0izDIBvUo7ZuGIDOQvrk9CxIe9zHdvDUhVDqNcTM,4449
|
|
10
|
+
sap_ecs_log_forwarder/json_logging.py,sha256=2kLmBfhq-qlj0Lmm1fPKz6qDLFJxV0Ga37JklrYFrL4,1308
|
|
11
|
+
sap_ecs_log_forwarder/metrics.py,sha256=XkEBsVA9RuLucEU_8TCdeNSrJXB3dNThGsgubVkyZiw,871
|
|
12
|
+
sap_ecs_log_forwarder/processor.py,sha256=XJbTkBlsg4B7O3yBzWWDtPPQd-q2MjkelvLueOQZ2dc,4079
|
|
13
|
+
sap_ecs_log_forwarder/utils.py,sha256=mVKEEjUfWjQGBn4OsvIDCeNysjEkU1x3loJ1A8IMCKs,852
|
|
14
|
+
sap_ecs_log_forwarder-1.0.0.dist-info/LICENSE,sha256=RTHTDJe35fXCEJkg3wT9DHAO8QQZWRNd9NELcyj4jN0,13528
|
|
15
|
+
sap_ecs_log_forwarder-1.0.0.dist-info/METADATA,sha256=NrEXiveIzr6qwo92sDrCouYYdsNXUx8r9PzvTBG0ssk,20196
|
|
16
|
+
sap_ecs_log_forwarder-1.0.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
17
|
+
sap_ecs_log_forwarder-1.0.0.dist-info/entry_points.txt,sha256=NHEVCnnKA3LQ-vynlLo2sf9hOGhbNrricddmwY8JQeA,129
|
|
18
|
+
sap_ecs_log_forwarder-1.0.0.dist-info/RECORD,,
|