sap-ecs-log-forwarder 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,634 @@
1
+ Metadata-Version: 2.1
2
+ Name: sap-ecs-log-forwarder
3
+ Version: 1.0.0
4
+ Summary: A package to consume events from an AWS SQS queue, Azure Storage Account Topic and GCP PubSub Topic, process log files, and forward them to a HTTP endpoint or file.
5
+ Home-page: https://www.sap.com/
6
+ License: SAP DEVELOPER LICENSE AGREEMENT
7
+ Keywords: SAP AWS Log Forwarder,SAP Azure Log Forwarder,SAP GCP Log Forwarder,SAP Log Forwarder
8
+ Author: SAP SE
9
+ Requires-Python: >=3.10,<4.0
10
+ Classifier: Development Status :: 5 - Production/Stable
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: License :: Other/Proprietary License
13
+ Classifier: Operating System :: MacOS :: MacOS X
14
+ Classifier: Operating System :: POSIX :: Linux
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Programming Language :: Python :: 3.14
21
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
22
+ Requires-Dist: aiohttp (>=3.13.2,<4.0.0)
23
+ Requires-Dist: azure-storage-blob (>=12.27.1,<13.0.0)
24
+ Requires-Dist: azure-storage-queue (>=12.14.1,<13.0.0)
25
+ Requires-Dist: boto3 (>=1.17.49,<2.0.0)
26
+ Requires-Dist: click (>=8.3.0,<9.0.0)
27
+ Requires-Dist: cryptography (>=44.0.1,<45.0.0)
28
+ Requires-Dist: google-cloud-pubsub (>=2.33.0,<3.0.0)
29
+ Requires-Dist: google-cloud-storage (>=3.6.0,<4.0.0)
30
+ Requires-Dist: requests (>=2.32.4,<3.0.0)
31
+ Requires-Dist: urllib3 (>=2.5.0,<3.0.0)
32
+ Description-Content-Type: text/markdown
33
+
34
+ # SAP ECS Log Forwarder
35
+
36
+ Unified Python service that consumes object / blob creation events from:
37
+ - AWS SQS (S3 ObjectCreated notifications)
38
+ - GCP Pub/Sub (Cloud Storage OBJECT_FINALIZE)
39
+ - Azure Queue (Storage BlobCreated events)
40
+
41
+ Downloads referenced log files, decompresses if gzip, splits into lines, and forwards each line to configured outputs (HTTP, files, console). Provides structured JSON logging, in‑memory metrics, regex filtering, encrypted credentials, jittered retries, and a configuration CLI.
42
+
43
+ ## Features
44
+ - Single `config.json` drives all cloud provider inputs.
45
+ - Per input regex filters: include / exclude.
46
+ - Output‑level filters (include / exclude) per output.
47
+ - Outputs: http, files, console (multiple per input).
48
+ - HTTP output supports TLS client certs, custom CA, and insecure skip verify.
49
+ - Encrypted credentials stored inline (Fernet) using `enc:` prefix.
50
+ - Structured JSON logging (console by default, optional file target).
51
+ - In‑memory counters periodically logged; daily reset at 00:00 UTC.
52
+ - Jittered exponential retries for transient failures.
53
+ - CLI to manage inputs, outputs, credentials, and log file path.
54
+ - Unit test scaffolding (pytest).
55
+
56
+ ## Architecture
57
+
58
+ ### AWS
59
+
60
+ ![Image for AWS on sap-ecs-log-forwarder](https://raw.githubusercontent.com/sap-ecs-log-forwarders/pypi/main/aws-log-forwarder.png)
61
+
62
+ ### Azure
63
+
64
+ ![Image for Azure on sap-ecs-log-forwarder](https://raw.githubusercontent.com/sap-ecs-log-forwarders/pypi/main/azure-log-forwarder.png)
65
+
66
+ ### GCP
67
+
68
+ ![Image for GCP on sap-ecs-log-forwarder](https://raw.githubusercontent.com/sap-ecs-log-forwarders/pypi/main/gcp-log-forwarder.png)
69
+
70
+ ## Configuration Schema (config.json)
71
+ ```json
72
+ {
73
+ "logLevel": "INFO",
74
+ "logFile": "/var/log/sap-log-forwarder/app.jsonl",
75
+ "inputs": [
76
+ {
77
+ "provider": "aws",
78
+ "name": "aws1",
79
+ "queue": "https://sqs.us-east-1.amazonaws.com/123/queue",
80
+ "region": "us-east-1",
81
+ "bucket": "my-bucket",
82
+ "includeFilter": ["\\.log$"],
83
+ "excludeFilter": ["debug"],
84
+ "maxRetries": 5,
85
+ "retryDelay": 10,
86
+ "authentication": {
87
+ "accessKeyId": "enc:...",
88
+ "secretAccessKey": "enc:...",
89
+ "encrypted": true
90
+ },
91
+ "outputs": [
92
+ {
93
+ "type": "http",
94
+ "destination": "https://example.com/ingest",
95
+ "authorization": {
96
+ "type": "bearer",
97
+ "token": "enc:...",
98
+ "encrypted": true
99
+ },
100
+ "tls": {
101
+ "pathToClientCert": "/path/client.crt",
102
+ "pathToClientKey": "/path/client.key",
103
+ "pathToCACert": "/path/ca.crt",
104
+ "insecureSkipVerify": false
105
+ },
106
+ "includeFilter": ["prod"],
107
+ "excludeFilter": ["test"]
108
+ },
109
+ {
110
+ "type": "files",
111
+ "destination": "logs/",
112
+ "compress": true,
113
+ "includeFilter": [],
114
+ "excludeFilter": []
115
+ },
116
+ {
117
+ "type": "console"
118
+ }
119
+ ]
120
+ }
121
+ ]
122
+ }
123
+ ```
124
+
125
+ Provider authentication fields:
126
+ - AWS: accessKeyId, secretAccessKey; or dynamic clientId, clientSecret, loginUrl, awsCredsUrl
127
+ - Azure: sasToken; or dynamic clientId, clientSecret, loginUrl, credsUrl
128
+ - GCP: serviceAccountJson (raw JSON string encrypted)
129
+ All optionally encrypted with `enc:` prefix.
130
+
131
+ ## Installation
132
+ ### With direct internet access:
133
+ ```sh
134
+ pip install sap-ecs-log-forwarder
135
+
136
+ or
137
+
138
+ pip install sap-ecs-log-forwarder==<version>
139
+ ```
140
+
141
+ ### Without direct internet access:
142
+ 1. **Download the wheel**
143
+ On a machine with internet access, visit the [PyPI files page](https://pypi.org/project/sap-ecs-log-forwarder/#files) for `sap-ecs-aws-log-forwarder` and download the `.whl` file matching your Python version (e.g., `sap_ecs_log_forwarder-1.0.0-py3-none-any.whl`).
144
+ 2. **Transfer the file**
145
+ Copy the downloaded wheel to the target (offline) machine using SCP, or another secure method.
146
+ 3. **Install from file**
147
+ On the offline machine, run:
148
+ ```sh
149
+ pip install /path/to/sap_ecs_log_forwarder-<version>-py3-none-any.whl
150
+ ```
151
+
152
+ ## Encryption
153
+ Generate a Fernet key:
154
+ ```python
155
+ from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())
156
+ ```
157
+ Export:
158
+ ```bash
159
+ export FORWARDER_ENCRYPTION_KEY="YOUR_FERNET_KEY"
160
+ ```
161
+ CLI will encrypt secrets you enter; encrypted values stored as `enc:<ciphertext>`.
162
+
163
+ ## CLI Usage
164
+ Add an input:
165
+ ```bash
166
+ sap-ecs-config-cli input add --provider aws --name aws1 --queue https://sqs... --region us-east-1 --bucket my-bucket
167
+ ```
168
+
169
+ ```bash
170
+ sap-ecs-config-cli input add --provider azure --name azure1 --queue my-queue --storage-account my-storage-account
171
+ ```
172
+
173
+ List inputs:
174
+ ```bash
175
+ sap-ecs-config-cli input list
176
+ ```
177
+
178
+ Remove input:
179
+ ```bash
180
+ sap-ecs-config-cli input remove aws1
181
+ ```
182
+
183
+ Add output (with optional output-level filters):
184
+ ```bash
185
+ sap-ecs-config-cli output add --input-name aws1 --type http --destination https://example.com/ingest \
186
+ --include prod --exclude test
187
+ ```
188
+
189
+ List outputs:
190
+ ```bash
191
+ sap-ecs-config-cli output list aws1
192
+ ```
193
+
194
+ Set HTTP authorization (encrypted):
195
+ ```bash
196
+ sap-ecs-config-cli creds set-http-auth --input-name aws1 --output-index 0 --auth-type bearer
197
+ ```
198
+
199
+ Set provider credentials (encrypted):
200
+ ```bash
201
+ sap-ecs-config-cli creds set-provider-auth --input-name aws1
202
+ ```
203
+
204
+ Configure JSON log file path:
205
+ ```bash
206
+ sap-ecs-config-cli set-log-file --path /var/log/sap-log-forwarder/app.jsonl
207
+ # To disable file logging:
208
+ sap-ecs-config-cli set-log-file --path ""
209
+ ```
210
+
211
+ ## Configuration File Location
212
+
213
+ Resolution order:
214
+ 1. Environment variable `SAP_LOG_FORWARDER_CONFIG` if set (supports `~` expansion).
215
+ 2. `./config.json` in current working directory (legacy behavior) if it exists.
216
+ 3. Default: `~/.sapecslogforwarder/config.json` (directory auto-created).
217
+
218
+ Show path:
219
+ ```bash
220
+ sap-ecs-config-cli config-path
221
+ ```
222
+
223
+ Set custom path:
224
+ ```bash
225
+ export SAP_LOG_FORWARDER_CONFIG=/etc/sap-log-forwarder/config.json
226
+ ```
227
+
228
+ All CLI operations read/write the resolved path.
229
+
230
+ ## Usage
231
+
232
+ To run the Log Forwarder, use the following command:
233
+
234
+ ```sh
235
+ sap-ecs-log-forwarder
236
+ ```
237
+
238
+ Service starts threads / async loops per input, logs metrics snapshot every 30s.
239
+
240
+ ## Structured Logging
241
+ Emits JSON lines to console (and to logFile if configured):
242
+ ```json
243
+ {"ts":"2025-11-25T12:00:00","level":"INFO","message":"Wrote logs to logs/app.log.gz","thread":"aws-aws1"}
244
+ ```
245
+
246
+ ## Metrics (logged)
247
+ Counters (examples):
248
+ - files_forward_success / files_forward_error
249
+ - http_forward_success / http_forward_error
250
+ - output_invocations
251
+ - aws_messages_processed / aws_retry
252
+ - gcp_messages_processed / gcp_retry
253
+ - azure_messages_processed / azure_retry
254
+
255
+ Snapshot logged periodically; can be extended to expose HTTP endpoint.
256
+
257
+ ## Filtering
258
+ - Include / exclude lists are regex patterns applied (case-insensitive) to object/blob/file names.
259
+ - Exclude overrides include.
260
+ - Output-level include/exclude filters are also supported per output.
261
+
262
+ ## Retries
263
+ Exponential backoff with added random jitter up to base retryDelay for download / processing failures.
264
+
265
+ ## HTTP Output
266
+ - Per-line POST with `Content-Type: application/json`.
267
+ - Auth types: bearer, api-key, basic.
268
+ - TLS options:
269
+ - pathToClientCert + pathToClientKey for mTLS
270
+ - pathToCACert for custom CA
271
+ - insecureSkipVerify to skip verification (not recommended)
272
+
273
+ ## Extending
274
+ - Add new output: implement handler and register in `processor.py` (OUTPUT_HANDLERS).
275
+ - Add metrics: `metrics.inc("<name>")`.
276
+ - Add provider: create runner class following existing pattern and register in `consumer.PROVIDERS`.
277
+
278
+ ## Security Notes
279
+ - Ensure `FORWARDER_ENCRYPTION_KEY` is injected via secrets manager.
280
+ - Structured logs avoid sensitive fields.
281
+
282
+ ### AWS Dynamic Authentication (Temporary Credentials)
283
+ Instead of static IAM keys, you can configure dynamic AWS auth. The forwarder will:
284
+ - POST to your backend login endpoint with `client_id` and `client_secret`, receive `session-id-raven` cookie.
285
+ - GET temporary AWS credentials for the target bucket, and refresh them every ~15 minutes or before expiration.
286
+
287
+ CLI:
288
+ ```bash
289
+ sap-ecs-config-cli creds set-provider-auth --input-name aws1
290
+ # Choose auth mode: dynamic
291
+ # Provide client_id, client_secret, login URL (default http://logserv.forwarder.host/api/v1/app/login)
292
+ # Provide AWS credentials URL (default http://logserv.forwarder.host/api/v1/aws/credentials)
293
+ ```
294
+
295
+ Config (snippet):
296
+ ```json
297
+ {
298
+ "provider": "aws",
299
+ "name": "aws1",
300
+ "queue": "https://sqs.us-east-1.amazonaws.com/123/queue",
301
+ "region": "us-east-1",
302
+ "bucket": "my-s3-bucket",
303
+ "authentication": {
304
+ "clientId": "enc:...",
305
+ "clientSecret": "enc:...",
306
+ "loginUrl": "http://logserv.forwarder.host/api/v1/app/login",
307
+ "awsCredsUrl": "http://logserv.forwarder.host/api/v1/aws/credentials",
308
+ "encrypted": true
309
+ }
310
+ }
311
+ ```
312
+
313
+ Backend contract:
314
+ ```bash
315
+ # Login
316
+ curl -X POST -H 'content-type: application/x-www-form-urlencoded' \
317
+ -d client_id=xxx -d client_secret=xxx \
318
+ http://logserv.forwarder.host/api/v1/app/login
319
+
320
+ # Fetch temporary creds (cookie required)
321
+ curl -H 'Cookie: session-id-raven=xxxx' \
322
+ 'http://logserv.forwarder.host/api/v1/aws/credentials?bucket=my-s3-bucket'
323
+ ```
324
+
325
+ Expected response:
326
+ ```json
327
+ {
328
+ "data": {
329
+ "AccessKeyId": "xxxx",
330
+ "SecretAccessKey": "xxxxx",
331
+ "SessionToken": "xxxxxx",
332
+ "Expiration": "2025-11-27T00:20:33Z",
333
+ "Region": "ap-south-1"
334
+ }
335
+ }
336
+ ```
337
+
338
+ Notes:
339
+ - Region from response overrides input region if provided; otherwise uses configured `region`.
340
+ - Static keys are used if dynamic fields are absent.
341
+ - Credentials refresh occurs on a schedule or 60s before expiration.
342
+
343
+ ### Azure Dynamic Authentication (Temporary SAS)
344
+ Instead of a static `sasToken`, you can configure dynamic Azure auth. The forwarder will:
345
+ - POST to your backend login endpoint with `client_id` and `client_secret`, receive `session-id-raven` cookie.
346
+ - GET temporary credentials for the storage account, extract `SASToken`, and refresh it every ~15 minutes (or before expiry).
347
+
348
+ CLI:
349
+ ```bash
350
+ sap-ecs-config-cli creds set-provider-auth --input-name azure1
351
+ # Choose auth mode: dynamic
352
+ # Provide client_id, client_secret, login URL (default http://logserv.forwarder.host/api/v1/app/login)
353
+ # Provide credentials URL (default http://logserv.forwarder.host/api/v1/azure/credentials)
354
+ ```
355
+
356
+ Config (snippet):
357
+ ```json
358
+ {
359
+ "provider": "azure",
360
+ "name": "azure1",
361
+ "queue": "my-queue",
362
+ "storageAccount": "mystorageacct",
363
+ "authentication": {
364
+ "clientId": "enc:...",
365
+ "clientSecret": "enc:...",
366
+ "loginUrl": "http://logserv.forwarder.host/api/v1/app/login",
367
+ "credsUrl": "http://logserv.forwarder.host/api/v1/azure/credentials",
368
+ "encrypted": true
369
+ }
370
+ }
371
+ ```
372
+
373
+ Notes:
374
+ - The credentials endpoint must accept `storage-account-name` query parameter.
375
+ - Response must include `data.SASToken` and `data.Expiration` (RFC3339 UTC).
376
+ - If `sasToken` is set and dynamic fields absent, static SAS is used.
377
+ - For connection strings, the runner uses them directly and skips SAS refresh.
378
+
379
+ ```bash
380
+ # macOS: test endpoints locally
381
+ curl -X POST -H "content-type: application/x-www-form-urlencoded" \
382
+ -d "client_id=xxx" -d "client_secret=xxx" \
383
+ http://logserv.forwarder.host/api/v1/app/login
384
+
385
+ curl -H "Cookie: session-id-raven=xxxxx" \
386
+ "http://logserv.forwarder.host/api/v1/azure/credentials?storage-account-name=mystorageacct"
387
+ ```
388
+
389
+ ### GCP Dynamic Authentication (Temporary Credentials)
390
+
391
+ **Will be available in a future release**
392
+
393
+ ## Systemd Deployment Guide
394
+ (For Linux servers only.)
395
+
396
+ ### Overview
397
+ Run the forwarder as a managed service with automatic restarts and persistent log files.
398
+
399
+ ### 1. Create a Service Account (Optional)
400
+ ```bash
401
+ sudo useradd -r -s /bin/false saplogfwd || true
402
+ ```
403
+
404
+ ### 2. Create Directories
405
+ ```bash
406
+ sudo mkdir -p /opt/sap-log-forwarder
407
+ sudo mkdir -p /var/log/sap-log-forwarder
408
+ sudo chown -R saplogfwd:saplogfwd /opt/sap-log-forwarder /var/log/sap-log-forwarder
409
+ ```
410
+
411
+ Place your project (editable install or wheel) under /opt/sap-log-forwarder or install via pip:
412
+ ```bash
413
+ cd /opt/sap-log-forwarder
414
+ sudo pip install sap-ecs-log-forwarder
415
+ ```
416
+
417
+ ### 3. Environment File
418
+ Create /etc/sap-log-forwarder.env:
419
+ ```bash
420
+ SAP_LOG_FORWARDER_CONFIG=/etc/sap-log-forwarder/config.json
421
+ FORWARDER_ENCRYPTION_KEY=YOUR_FERNET_KEY
422
+ PYTHONUNBUFFERED=1
423
+ LOG_LEVEL=INFO
424
+ ```
425
+
426
+ (Generate key earlier and replace.)
427
+
428
+ ### 4. Systemd Unit
429
+ Create /etc/systemd/system/sap-log-forwarder.service:
430
+ ```ini
431
+ [Unit]
432
+ Description=SAP ECS Log Forwarder
433
+ After=network.target
434
+ Wants=network-online.target
435
+
436
+ [Service]
437
+ Type=simple
438
+ User=saplogfwd
439
+ Group=saplogfwd
440
+ WorkingDirectory=/opt/sap-log-forwarder
441
+ EnvironmentFile=/etc/sap-log-forwarder.env
442
+ ExecStart=/usr/bin/python -m sap_ecs_log_forwarder.consumer
443
+ Restart=always
444
+ RestartSec=5
445
+ # File logging (append). Systemd 240+ supports append:
446
+ StandardOutput=append:/var/log/sap-log-forwarder/forwarder.out.log
447
+ StandardError=append:/var/log/sap-log-forwarder/forwarder.err.log
448
+ # Alternatively (recommended) use journald and remove StandardOutput/StandardError lines.
449
+
450
+ # Resource limits (optional)
451
+ NoNewPrivileges=yes
452
+ ProtectSystem=full
453
+ ProtectHome=yes
454
+
455
+ [Install]
456
+ WantedBy=multi-user.target
457
+ ```
458
+
459
+ Reload and start:
460
+ ```bash
461
+ sudo systemctl daemon-reload
462
+ sudo systemctl enable --now sap-log-forwarder
463
+ ```
464
+
465
+ Status:
466
+ ```bash
467
+ systemctl status sap-log-forwarder
468
+ ```
469
+
470
+ ### 5. Log Rotation (If Using File Output)
471
+ Create /etc/logrotate.d/sap-log-forwarder:
472
+ ```conf
473
+ /var/log/sap-log-forwarder/forwarder.*.log {
474
+ daily
475
+ rotate 14
476
+ compress
477
+ missingok
478
+ notifempty
479
+ create 0640 saplogfwd saplogfwd
480
+ sharedscripts
481
+ postrotate
482
+ systemctl kill -s SIGUSR1 sap-log-forwarder || true
483
+ endscript
484
+ }
485
+ ```
486
+
487
+ (Forwarder ignores SIGUSR1 now; postrotate just optional. Compression handled by logrotate.)
488
+
489
+ ### 6. Using Journald Instead
490
+ Remove StandardOutput/StandardError lines to keep logs in journal:
491
+ ```bash
492
+ journalctl -u sap-log-forwarder -f
493
+ ```
494
+ To export periodically, use:
495
+ ```bash
496
+ journalctl -u sap-log-forwarder --since "1 hour ago" > /var/log/sap-log-forwarder/hour.log
497
+ ```
498
+
499
+ ### 7. Config File Location
500
+ Place config.json in WorkingDirectory (/opt/sap-log-forwarder/config.json). Edit via CLI:
501
+ ```bash
502
+ sudo -u saplogfwd sap-ecs-config-cli input list
503
+ ```
504
+
505
+ ### 8. Permissions
506
+ Ensure /var/log/sap-log-forwarder writable by service user. If using file outputs (destination logs/):
507
+ ```bash
508
+ sudo mkdir -p /opt/sap-log-forwarder/logs
509
+ sudo chown saplogfwd:saplogfwd /opt/sap-log-forwarder/logs
510
+ ```
511
+
512
+ ### 9. Updating
513
+ ```bash
514
+ sudo systemctl stop sap-log-forwarder
515
+ sudo pip install --upgrade sap-ecs-log-forwarder
516
+ sudo systemctl start sap-log-forwarder
517
+ ```
518
+
519
+ ### 10. Common Checks
520
+ ```bash
521
+ systemctl status sap-log-forwarder
522
+ journalctl -u sap-log-forwarder -n 50
523
+ ls -lh /var/log/sap-log-forwarder
524
+ ```
525
+
526
+ ### 11. Troubleshooting Restart Loops
527
+ - Inspect recent logs (`journalctl -u sap-log-forwarder -f`).
528
+ - Validate config.json structure.
529
+ - Verify FORWARDER_ENCRYPTION_KEY present (`systemctl show -p Environment sap-log-forwarder`).
530
+
531
+ ### 12. Security Hardening (Optional)
532
+ Add directives:
533
+ ```
534
+ PrivateTmp=yes
535
+ ProtectKernelTunables=yes
536
+ ProtectKernelModules=yes
537
+ RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
538
+ ```
539
+
540
+ Keep encryption key in a root-readable-only environment file (chmod 640).
541
+
542
+ ### 13. Switching To Virtualenv
543
+ If installed in virtualenv:
544
+ ```
545
+ ExecStart=/opt/sap-log-forwarder/venv/bin/python -m sap_ecs_log_forwarder.consumer
546
+ ```
547
+
548
+ Ensure venv ownership matches service user.
549
+
550
+ ### 14. Testing Manual Run
551
+ ```bash
552
+ sudo -u saplogfwd FORWARDER_ENCRYPTION_KEY=YOUR_FERNET_KEY python -m sap_ecs_log_forwarder.consumer
553
+ ```
554
+
555
+ Stop with:
556
+ ```bash
557
+ sudo systemctl stop sap-log-forwarder
558
+ ```
559
+
560
+ Service now runs with automatic restart (Restart=always) and logs persisted under /var/log/sap-log-forwarder.
561
+
562
+ ## Troubleshooting
563
+
564
+ ### Quick Checklist
565
+ 1. config.json present and valid (`jq '.' config.json`).
566
+ 2. `FORWARDER_ENCRYPTION_KEY` exported.
567
+ 3. Inputs listed (`sap-ecs-config-cli input list`).
568
+ 4. Outputs configured (`sap-ecs-config-cli output list <input>`).
569
+ 5. Per-input `logLevel` set (use `DEBUG` for diagnosis).
570
+ 6. If using HTTP/TLS, verify cert/key/CA paths.
571
+
572
+ ### General Issues
573
+ | Symptom | Cause | Fix |
574
+ |---------|-------|-----|
575
+ | Encrypted values not decrypted | Missing `FORWARDER_ENCRYPTION_KEY` | Export key before running consumer |
576
+ | No files processed | Check exclude/include filters | Adjust filters |
577
+ | Output skipped | Output-level include/exclude filters | Remove or correct filters |
578
+ | HTTP send fails | Bad URL, TLS paths, undecrypted auth | Verify destination, cert/key/CA paths, export key |
579
+ | File output missing | No destination or permission denied | Provide writable directory in output config |
580
+
581
+ ### AWS (SQS + S3)
582
+ - No messages: Ensure S3 event notification configured (ObjectCreated -> SQS).
583
+ - Objects ignored: Bucket mismatch, eventName not `ObjectCreated:Put`, regex filters.
584
+ - Repeated retries (`aws_retry`): Missing `s3:GetObject` or corrupt gzip.
585
+ - Credential failure: Re-run `creds set-provider-auth` and export key.
586
+
587
+ ### GCP (Pub/Sub + Storage)
588
+ - Subscription path invalid: Must be `projects/<project>/subscriptions/<name>`.
589
+ - PermissionDenied: Grant `roles/pubsub.subscriber` and `roles/storage.objectViewer`.
590
+ - Service account JSON parse failure: Re-enter credentials (file/paste mode).
591
+ - Events ignored: Not `OBJECT_FINALIZE` or filters reject.
592
+
593
+ ### Azure (Queue + Blob)
594
+ - Decode errors: Ensure Event Grid -> Queue message schema.
595
+ - Missing blob URL: Recreate Event Grid subscription; `data.url` must exist.
596
+ - Unauthorized blob download: Provide SAS token or full connection string.
597
+ - Filters miss: Subject pattern `/blobServices/default/containers/<c>/blobs/<name>`; adjust regex.
598
+
599
+ ### Performance
600
+ - Slow: Large object fully loaded; consider future streaming enhancement.
601
+ - Many retries: Increase `retryDelay`, verify network and permissions.
602
+ - Parallelism: Each input runs its own thread/loop; split workloads across inputs.
603
+
604
+ ### Debug Commands (macOS)
605
+ ```bash
606
+ env | grep FORWARDER_ENCRYPTION_KEY
607
+ sap-ecs-config-cli input list
608
+ sap-ecs-config-cli output list <input-name>
609
+ python -m sap_ecs_log_forwarder.consumer
610
+ ```
611
+
612
+ ### Metrics Reference
613
+ - aws_messages_processed / aws_retry
614
+ - gcp_messages_processed / gcp_retry
615
+ - azure_messages_processed / azure_retry
616
+ - output_invocations
617
+ - files_forward_success / files_forward_error
618
+ - http_forward_success / http_forward_error
619
+
620
+ ### When Nothing Processes
621
+ 1. Set input `logLevel` to DEBUG.
622
+ 2. Temporarily clear include/exclude filters.
623
+ 3. Upload a small test file that matches expected pattern.
624
+ 4. Confirm metrics counters increment.
625
+ 5. Reapply filters gradually.
626
+
627
+ ## License
628
+ This application and its source code are licensed under the terms of the SAP Developer License Agreement. See the LICENSE file for more information.
629
+
630
+ ## Changelog
631
+
632
+ **Version 1.0.0**
633
+
634
+ - First proper release
@@ -0,0 +1,18 @@
1
+ sap_ecs_log_forwarder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
+ sap_ecs_log_forwarder/aws.py,sha256=1DXtV1aAbmciCxoU3T99nXPH0cXIbCu1DOV85j-RpsU,10149
3
+ sap_ecs_log_forwarder/azure.py,sha256=ZeOltwwZHQXVIZZ7TKzO4px5VGFqnH3kAYKWRwhP0Z0,12099
4
+ sap_ecs_log_forwarder/base_runner.py,sha256=PVCpKcUtRO5v-_7oD5kN6Ty-P5URwesU17_BjraHB3c,586
5
+ sap_ecs_log_forwarder/cli.py,sha256=MmPTP0UDtqlPQEYA0lRy_7qeTFne8dQIajxddhABYHQ,11718
6
+ sap_ecs_log_forwarder/config.py,sha256=_zr48_xDn2XEPwpNnQelaBBZXU5SVa6cJPAzNTaM9hk,3446
7
+ sap_ecs_log_forwarder/consumer.py,sha256=ODdlThC-yzi1A5MdZcrlaoWiIC9y9C2R7hEoTCxh2vI,1738
8
+ sap_ecs_log_forwarder/crypto.py,sha256=MCn2K3PLecRdmtW-6yvT-5hlLHWdVhNUH83OlqYNitA,1104
9
+ sap_ecs_log_forwarder/gcp.py,sha256=0MP0izDIBvUo7ZuGIDOQvrk9CxIe9zHdvDUhVDqNcTM,4449
10
+ sap_ecs_log_forwarder/json_logging.py,sha256=2kLmBfhq-qlj0Lmm1fPKz6qDLFJxV0Ga37JklrYFrL4,1308
11
+ sap_ecs_log_forwarder/metrics.py,sha256=XkEBsVA9RuLucEU_8TCdeNSrJXB3dNThGsgubVkyZiw,871
12
+ sap_ecs_log_forwarder/processor.py,sha256=XJbTkBlsg4B7O3yBzWWDtPPQd-q2MjkelvLueOQZ2dc,4079
13
+ sap_ecs_log_forwarder/utils.py,sha256=mVKEEjUfWjQGBn4OsvIDCeNysjEkU1x3loJ1A8IMCKs,852
14
+ sap_ecs_log_forwarder-1.0.0.dist-info/LICENSE,sha256=RTHTDJe35fXCEJkg3wT9DHAO8QQZWRNd9NELcyj4jN0,13528
15
+ sap_ecs_log_forwarder-1.0.0.dist-info/METADATA,sha256=NrEXiveIzr6qwo92sDrCouYYdsNXUx8r9PzvTBG0ssk,20196
16
+ sap_ecs_log_forwarder-1.0.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
17
+ sap_ecs_log_forwarder-1.0.0.dist-info/entry_points.txt,sha256=NHEVCnnKA3LQ-vynlLo2sf9hOGhbNrricddmwY8JQeA,129
18
+ sap_ecs_log_forwarder-1.0.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: poetry-core 1.9.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,4 @@
1
+ [console_scripts]
2
+ sap-ecs-config-cli=sap_ecs_log_forwarder.cli:cli
3
+ sap-ecs-log-forwarder=sap_ecs_log_forwarder.consumer:run_all
4
+