log-foundry 0.1.0__tar.gz → 0.1.1.dev2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {log_foundry-0.1.0 → log_foundry-0.1.1.dev2}/PKG-INFO +114 -69
- {log_foundry-0.1.0 → log_foundry-0.1.1.dev2}/README.md +113 -68
- {log_foundry-0.1.0 → log_foundry-0.1.1.dev2}/pyproject.toml +7 -6
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/__init__.py +6 -6
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/api.py +7 -7
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/config.py +3 -3
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/console.py +1 -1
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/context.py +4 -4
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/decorator.py +5 -5
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/model.py +2 -2
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/_socket.py +1 -1
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/callback.py +2 -2
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/clickhouse.py +3 -3
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/datadog.py +3 -3
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/elasticsearch.py +3 -3
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/eventhubs.py +3 -3
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/file.py +2 -2
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/filtering.py +2 -2
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/firehose.py +5 -5
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/honeycomb.py +2 -2
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/http.py +2 -2
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/kafka.py +2 -2
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/kinesis.py +5 -5
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/logging_sink.py +2 -2
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/logstash.py +4 -4
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/loki.py +3 -3
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/mongodb.py +4 -4
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/multi.py +4 -4
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/nats.py +2 -2
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/newrelic.py +2 -2
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/postgres.py +3 -3
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/pubsub.py +2 -2
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/rabbitmq.py +2 -2
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/redis.py +1 -1
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/sentry.py +7 -7
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/sns.py +4 -4
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/splunk.py +4 -4
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/sqlite.py +1 -1
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/sqs.py +4 -4
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/syslog.py +5 -5
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/transform.py +2 -2
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/util.py +4 -4
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/worker.py +3 -3
- {log_foundry-0.1.0 → log_foundry-0.1.1.dev2}/LICENSE +0 -0
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/ids.py +0 -0
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/py.typed +0 -0
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/__init__.py +0 -0
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/_chunk.py +0 -0
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/_time.py +0 -0
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/base.py +0 -0
- {log_foundry-0.1.0/src/log_forge → log_foundry-0.1.1.dev2/src/log_foundry}/sinks/stdout.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: log-foundry
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.1.dev2
|
|
4
4
|
Summary: Generate logs for your console and JSON events for downstream consumption.
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -33,13 +33,13 @@ Requires-Dist: redis (>=5.0) ; extra == "redis"
|
|
|
33
33
|
Requires-Dist: sentry-sdk (>=2.0) ; extra == "sentry"
|
|
34
34
|
Description-Content-Type: text/markdown
|
|
35
35
|
|
|
36
|
-
# log-
|
|
36
|
+
# log-foundry
|
|
37
37
|
|
|
38
38
|
Consistent, structured (JSON) logs for every decorated function call — correlated by shared
|
|
39
39
|
trace/span IDs, ready to ship to any of 30-plus built-in sinks (stdout by default; SQS → ELK is
|
|
40
40
|
the headline production path).
|
|
41
41
|
|
|
42
|
-
`log-
|
|
42
|
+
`log-foundry` owns the **logs** pillar of observability. You decorate a function with `@trace`;
|
|
43
43
|
it emits one identically-shaped JSON record when the call starts and another when it ends
|
|
44
44
|
(with duration and status), stitched together by W3C-compatible trace and span IDs so nested
|
|
45
45
|
calls form a tree you can query later.
|
|
@@ -60,27 +60,42 @@ calls form a tree you can query later.
|
|
|
60
60
|
|
|
61
61
|
## Installation
|
|
62
62
|
|
|
63
|
-
`log-
|
|
63
|
+
Published on PyPI as **[`log-foundry`](https://pypi.org/project/log-foundry/)**:
|
|
64
64
|
|
|
65
65
|
```bash
|
|
66
|
-
|
|
67
|
-
|
|
66
|
+
pip install log-foundry # core, zero dependencies
|
|
67
|
+
pip install 'log-foundry[aws]' # + boto3 for the SQS/SNS/Kinesis/Firehose sinks
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
> **Renamed in 0.2.0: `log_forge` → `log_foundry`.** The import package now matches the
|
|
71
|
+
> distribution name — `pip install log-foundry`, then `import log_foundry`. If you are on
|
|
72
|
+
> `0.1.x`, update your imports; there is no compatibility shim. The project was originally
|
|
73
|
+
> called *log-forge*, but PyPI rejects that name as too similar to the unrelated, pre-existing
|
|
74
|
+
> [`logforge`](https://pypi.org/project/logforge/) project — its similarity check collapses
|
|
75
|
+
> separators, so `log-forge` and `logforge` count as the same name. Rather than keep a
|
|
76
|
+
> distribution and an import name that disagreed, everything is now `log-foundry` /
|
|
77
|
+
> `log_foundry`.
|
|
78
|
+
>
|
|
79
|
+
> Migrating from `0.1.x` is a find-and-replace on `log_forge` → `log_foundry`; no module moved
|
|
80
|
+
> and no public API changed. A handful of *emitted* defaults carry the name and shift with it:
|
|
81
|
+
> `LoggingSink`'s default logger (`logging.getLogger("log_foundry")`), `SyslogSink(app_name=…)`,
|
|
82
|
+
> `SplunkHECSink(source=…)`, Datadog's `ddsource`, and Sentry's client tag. Override them
|
|
83
|
+
> explicitly if a downstream query or dashboard pins the old string.
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
import log_foundry
|
|
68
87
|
|
|
69
|
-
|
|
70
|
-
poetry install -E aws # or: pip install '.[aws]'
|
|
88
|
+
print(log_foundry.__version__) # the installed version
|
|
71
89
|
```
|
|
72
90
|
|
|
73
|
-
|
|
91
|
+
To work on the library itself, install from a clone:
|
|
74
92
|
|
|
75
93
|
```bash
|
|
76
|
-
|
|
77
|
-
|
|
94
|
+
# the version is derived from Git tags, so clone with history (not --depth 1)
|
|
95
|
+
poetry self add "poetry-dynamic-versioning[plugin]" # one-time, resolves the version locally
|
|
96
|
+
poetry install --with dev # or: pip install -e .
|
|
78
97
|
```
|
|
79
98
|
|
|
80
|
-
> **Note the name split.** The distribution is **`log-foundry`** on PyPI, but the import name
|
|
81
|
-
> stays **`log_forge`** — `pip install log-foundry`, then `import log_forge`. PyPI rejects
|
|
82
|
-
> `log-forge` as too similar to the unrelated, pre-existing `logforge` project.
|
|
83
|
-
|
|
84
99
|
### Optional extras
|
|
85
100
|
|
|
86
101
|
The core is dependency-free. Each sink built on a third-party client lives behind its own extra
|
|
@@ -105,7 +120,7 @@ Loki, Logstash, Syslog, Datadog, Splunk, New Relic, Honeycomb) — need **no** e
|
|
|
105
120
|
## Quickstart
|
|
106
121
|
|
|
107
122
|
```python
|
|
108
|
-
import
|
|
123
|
+
import log_foundry as lf
|
|
109
124
|
|
|
110
125
|
# Call once at startup. These values are stamped onto every event.
|
|
111
126
|
lf.configure(service="billing-api", version="1.4.2", env="prod")
|
|
@@ -137,7 +152,7 @@ with the child span pointing at its parent via `parent_span_id`:
|
|
|
137
152
|
|
|
138
153
|
## How it works
|
|
139
154
|
|
|
140
|
-

|
|
141
156
|
|
|
142
157
|
A traced call travels through a small pipeline. The first four steps run on your own thread
|
|
143
158
|
and are deliberately fast; the last two run on a background thread so your code never waits on
|
|
@@ -289,13 +304,13 @@ class Sink(Protocol):
|
|
|
289
304
|
|
|
290
305
|
Wire one up by passing an instance to `configure(sink=...)`; if you never do, the first decorated
|
|
291
306
|
call falls back to `StdoutSink()`. Sinks are **not** re-exported at the top level — import each
|
|
292
|
-
from its own module, e.g. `from
|
|
307
|
+
from its own module, e.g. `from log_foundry.sinks.sqs import SQSSink`.
|
|
293
308
|
|
|
294
309
|
A few conventions hold across every sink below:
|
|
295
310
|
|
|
296
311
|
- **Extras.** The core is dependency-free. A sink built on a third-party client sits behind the
|
|
297
312
|
optional extra named in its table (blank = zero-dependency, stdlib only); the client is imported
|
|
298
|
-
lazily, so `import
|
|
313
|
+
lazily, so `import log_foundry.sinks.<x>` never fails for a missing dependency — only *constructing*
|
|
299
314
|
the sink without an injected client does. See [Optional extras](#optional-extras).
|
|
300
315
|
- **Injection.** Sinks backed by an external resource accept an injected client/connection/stream
|
|
301
316
|
(`client=`, `connection=`, `producer=`, `stream=`, `opener=`) for testing or bespoke configuration.
|
|
@@ -309,13 +324,13 @@ A few conventions hold across every sink below:
|
|
|
309
324
|
|
|
310
325
|
| Sink | Import from | Configure |
|
|
311
326
|
|---|---|---|
|
|
312
|
-
| `StdoutSink` | `
|
|
313
|
-
| `StderrSink` | `
|
|
314
|
-
| `NullSink` | `
|
|
315
|
-
| `MemorySink` | `
|
|
327
|
+
| `StdoutSink` | `log_foundry.sinks.stdout` | `StdoutSink(stream=sys.stdout)` — one JSON line per event; the zero-config default |
|
|
328
|
+
| `StderrSink` | `log_foundry.sinks.util` | `StderrSink(stream=sys.stderr)` — same, on stderr (twelve-factor) |
|
|
329
|
+
| `NullSink` | `log_foundry.sinks.util` | `NullSink()` — discard everything; `.dropped` counts events |
|
|
330
|
+
| `MemorySink` | `log_foundry.sinks.util` | `MemorySink(maxlen=None)` — collect into `.events` (a bounded ring when `maxlen` is set) |
|
|
316
331
|
|
|
317
332
|
```python
|
|
318
|
-
from
|
|
333
|
+
from log_foundry.sinks.stdout import StdoutSink
|
|
319
334
|
lf.configure(sink=StdoutSink()) # explicit; also the zero-config default
|
|
320
335
|
```
|
|
321
336
|
|
|
@@ -326,16 +341,16 @@ to a plain callable.
|
|
|
326
341
|
|
|
327
342
|
| Sink | Import from | Configure |
|
|
328
343
|
|---|---|---|
|
|
329
|
-
| `MultiSink` | `
|
|
330
|
-
| `FilteringSink` | `
|
|
331
|
-
| `TransformSink` | `
|
|
332
|
-
| `CallbackSink` | `
|
|
344
|
+
| `MultiSink` | `log_foundry.sinks.multi` | `MultiSink(*sinks)` — forward each batch to every child; a failing child is isolated and counted on `.failed` |
|
|
345
|
+
| `FilteringSink` | `log_foundry.sinks.filtering` | `FilteringSink(inner, *, predicate=None, min_level=None)` — forward only events passing `predicate` and/or at/above `min_level` |
|
|
346
|
+
| `TransformSink` | `log_foundry.sinks.transform` | `TransformSink(inner, fn)` — map each event through `fn` before forwarding; return `None` to drop one |
|
|
347
|
+
| `CallbackSink` | `log_foundry.sinks.callback` | `CallbackSink(fn, *, on_close=None)` — hand each batch to any callable |
|
|
333
348
|
|
|
334
349
|
```python
|
|
335
|
-
from
|
|
336
|
-
from
|
|
337
|
-
from
|
|
338
|
-
from
|
|
350
|
+
from log_foundry.sinks.multi import MultiSink
|
|
351
|
+
from log_foundry.sinks.filtering import FilteringSink
|
|
352
|
+
from log_foundry.sinks.stdout import StdoutSink
|
|
353
|
+
from log_foundry.sinks.sqs import SQSSink
|
|
339
354
|
|
|
340
355
|
lf.configure(sink=MultiSink(
|
|
341
356
|
StdoutSink(), # echo everything locally
|
|
@@ -350,9 +365,9 @@ level is unknown or missing fails open (is forwarded).
|
|
|
350
365
|
|
|
351
366
|
| Sink | Import from | Configure |
|
|
352
367
|
|---|---|---|
|
|
353
|
-
| `LoggingSink` | `
|
|
368
|
+
| `LoggingSink` | `log_foundry.sinks.logging_sink` | `LoggingSink(logger=None, *, default_level="INFO")` — emit each event as a `logging.LogRecord` |
|
|
354
369
|
|
|
355
|
-
Hands every event to a `logging.Logger` (default `logging.getLogger("
|
|
370
|
+
Hands every event to a `logging.Logger` (default `logging.getLogger("log_foundry")`) so your existing
|
|
356
371
|
handlers, formatters, and `logging.config` apply. Identity fields and the nested `fields` are
|
|
357
372
|
attached to each record; the sink never configures or tears down logging itself.
|
|
358
373
|
|
|
@@ -360,9 +375,9 @@ attached to each record; the sink never configures or tears down logging itself.
|
|
|
360
375
|
|
|
361
376
|
| Sink | Import from | Configure |
|
|
362
377
|
|---|---|---|
|
|
363
|
-
| `FileSink` | `
|
|
364
|
-
| `RotatingFileSink` | `
|
|
365
|
-
| `SQLiteSink` | `
|
|
378
|
+
| `FileSink` | `log_foundry.sinks.file` | `FileSink(path, *, encoding="utf-8")` — append NDJSON to one file |
|
|
379
|
+
| `RotatingFileSink` | `log_foundry.sinks.file` | `RotatingFileSink(path, *, max_bytes=0, backup_count=0, when=None, interval=1)` — rotate by size and/or time, keeping `backup_count` numbered backups |
|
|
380
|
+
| `SQLiteSink` | `log_foundry.sinks.sqlite` | `SQLiteSink(database, *, table="log_events", create_table=True)` — batch-insert into an embedded SQLite DB |
|
|
366
381
|
|
|
367
382
|
`RotatingFileSink`'s time trigger uses a `when` unit code — `"S"`/`"M"`/`"H"`/`"D"` — times `interval`
|
|
368
383
|
(either trigger, or both, can be enabled). `SQLiteSink` stores each event as full JSON plus projected
|
|
@@ -370,7 +385,7 @@ attached to each record; the sink never configures or tears down logging itself.
|
|
|
370
385
|
you provision the table yourself.
|
|
371
386
|
|
|
372
387
|
```python
|
|
373
|
-
from
|
|
388
|
+
from log_foundry.sinks.file import RotatingFileSink
|
|
374
389
|
lf.configure(sink=RotatingFileSink("app.log.jsonl", max_bytes=10_000_000, backup_count=5))
|
|
375
390
|
```
|
|
376
391
|
|
|
@@ -382,15 +397,15 @@ to `HTTPSink` (`headers=`, `auth=`, `gzip=`, `timeout=`, `max_retries=`).
|
|
|
382
397
|
|
|
383
398
|
| Sink | Import from | Configure |
|
|
384
399
|
|---|---|---|
|
|
385
|
-
| `HTTPSink` | `
|
|
386
|
-
| `ElasticsearchSink` | `
|
|
387
|
-
| `OpenSearchSink` | `
|
|
388
|
-
| `LokiSink` | `
|
|
389
|
-
| `LogstashSink` | `
|
|
390
|
-
| `SyslogSink` | `
|
|
400
|
+
| `HTTPSink` | `log_foundry.sinks.http` | `HTTPSink(url, *, method="POST", headers=None, auth=None, body_format="ndjson", timeout=5.0, gzip=False, max_retries=3)` — generic POST. `auth` is a bearer-token `str` or `(user, pass)` for basic; `body_format` is `"ndjson"` or `"json_array"` |
|
|
401
|
+
| `ElasticsearchSink` | `log_foundry.sinks.elasticsearch` | `ElasticsearchSink(url, *, index, auth=None, **http_kwargs)` — POST to `_bulk`, parsing per-item errors (`.item_errors`) |
|
|
402
|
+
| `OpenSearchSink` | `log_foundry.sinks.elasticsearch` | same signature as `ElasticsearchSink` (identical bulk protocol) |
|
|
403
|
+
| `LokiSink` | `log_foundry.sinks.loki` | `LokiSink(url, *, labels=("service", "env", "level"), **http_kwargs)` — Grafana Loki push API |
|
|
404
|
+
| `LogstashSink` | `log_foundry.sinks.logstash` | `LogstashSink(url=…, **http_kwargs)` for HTTP, **or** `LogstashSink(host=…, port=…, transport="tcp")` for a raw TCP/UDP socket |
|
|
405
|
+
| `SyslogSink` | `log_foundry.sinks.syslog` | `SyslogSink(host, port=514, *, transport="udp", facility="user", app_name="log-foundry")` — RFC 5424 over UDP/TCP |
|
|
391
406
|
|
|
392
407
|
```python
|
|
393
|
-
from
|
|
408
|
+
from log_foundry.sinks.elasticsearch import ElasticsearchSink
|
|
394
409
|
lf.configure(sink=ElasticsearchSink("https://es.internal:9200", index="app-logs",
|
|
395
410
|
auth=("elastic", "…")))
|
|
396
411
|
```
|
|
@@ -402,11 +417,11 @@ Also HTTP-based. All are zero-dependency **except** `SentrySink`, which prefers
|
|
|
402
417
|
|
|
403
418
|
| Sink | Import from | Extra | Configure |
|
|
404
419
|
|---|---|---|---|
|
|
405
|
-
| `DatadogSink` | `
|
|
406
|
-
| `SplunkHECSink` | `
|
|
407
|
-
| `NewRelicSink` | `
|
|
408
|
-
| `HoneycombSink` | `
|
|
409
|
-
| `SentrySink` | `
|
|
420
|
+
| `DatadogSink` | `log_foundry.sinks.datadog` | — | `DatadogSink(api_key, *, site="datadoghq.com", service=None, ddtags=None)` |
|
|
421
|
+
| `SplunkHECSink` | `log_foundry.sinks.splunk` | — | `SplunkHECSink(url, token, *, host=None, source="log-foundry")` — HTTP Event Collector |
|
|
422
|
+
| `NewRelicSink` | `log_foundry.sinks.newrelic` | — | `NewRelicSink(api_key, *, region="US")` — `region` is `"US"` or `"EU"` |
|
|
423
|
+
| `HoneycombSink` | `log_foundry.sinks.honeycomb` | — | `HoneycombSink(api_key, dataset, *, url="https://api.honeycomb.io")` |
|
|
424
|
+
| `SentrySink` | `log_foundry.sinks.sentry` | `sentry` | `SentrySink(dsn=None, *, min_level="ERROR")` — sends only `min_level`+ events |
|
|
410
425
|
|
|
411
426
|
With the `sentry` extra installed, `SentrySink` captures via `sentry_sdk.capture_event` (initialize
|
|
412
427
|
the SDK yourself with `sentry_sdk.init(...)`); without it, pass `dsn=` and events are POSTed as
|
|
@@ -415,19 +430,19 @@ Sentry envelopes over HTTP.
|
|
|
415
430
|
#### AWS — the durable-buffer path (`aws` extra)
|
|
416
431
|
|
|
417
432
|
`pip install 'log-foundry[aws]'` (pulls `boto3`). Credentials and region come from boto3's standard
|
|
418
|
-
chain — log-
|
|
433
|
+
chain — log-foundry adds none of its own. Each re-chunks every batch to the service's hard per-request
|
|
419
434
|
limits, retries partial failures, and drops any single event too large to ever fit (counted on
|
|
420
435
|
`.dropped_oversized`).
|
|
421
436
|
|
|
422
437
|
| Sink | Import from | Configure |
|
|
423
438
|
|---|---|---|
|
|
424
|
-
| `SQSSink` | `
|
|
425
|
-
| `SNSSink` | `
|
|
426
|
-
| `KinesisSink` | `
|
|
427
|
-
| `FirehoseSink` | `
|
|
439
|
+
| `SQSSink` | `log_foundry.sinks.sqs` | `SQSSink(queue_url, *, max_retries=3)` — the headline production path: a durable buffer in front of ELK, absorbing downstream spikes/outages |
|
|
440
|
+
| `SNSSink` | `log_foundry.sinks.sns` | `SNSSink(topic_arn, *, max_retries=3)` |
|
|
441
|
+
| `KinesisSink` | `log_foundry.sinks.kinesis` | `KinesisSink(stream_name, *, partition_key_field="trace_id", max_retries=3)` |
|
|
442
|
+
| `FirehoseSink` | `log_foundry.sinks.firehose` | `FirehoseSink(delivery_stream, *, max_retries=3)` |
|
|
428
443
|
|
|
429
444
|
```python
|
|
430
|
-
from
|
|
445
|
+
from log_foundry.sinks.sqs import SQSSink
|
|
431
446
|
lf.configure(service="payments",
|
|
432
447
|
sink=SQSSink(queue_url="https://sqs.us-east-1.amazonaws.com/123456789012/logs"))
|
|
433
448
|
```
|
|
@@ -440,16 +455,16 @@ Each needs its own extra (lazy-imported). All publish + retry within a bound and
|
|
|
440
455
|
|
|
441
456
|
| Sink | Import from | Extra | Configure |
|
|
442
457
|
|---|---|---|---|
|
|
443
|
-
| `KafkaSink` | `
|
|
444
|
-
| `RedisStreamsSink` | `
|
|
445
|
-
| `RedisListSink` | `
|
|
446
|
-
| `RabbitMQSink` | `
|
|
447
|
-
| `NATSSink` | `
|
|
448
|
-
| `GooglePubSubSink` | `
|
|
449
|
-
| `AzureEventHubsSink` | `
|
|
458
|
+
| `KafkaSink` | `log_foundry.sinks.kafka` | `kafka` | `KafkaSink(topic, *, bootstrap_servers="…", key_field="trace_id")` |
|
|
459
|
+
| `RedisStreamsSink` | `log_foundry.sinks.redis` | `redis` | `RedisStreamsSink(stream, *, url=None)` — `XADD` |
|
|
460
|
+
| `RedisListSink` | `log_foundry.sinks.redis` | `redis` | `RedisListSink(key, *, url=None)` — `RPUSH` |
|
|
461
|
+
| `RabbitMQSink` | `log_foundry.sinks.rabbitmq` | `amqp` | `RabbitMQSink(*, exchange, routing_key, url=None)` — persistent messages |
|
|
462
|
+
| `NATSSink` | `log_foundry.sinks.nats` | `nats` | `NATSSink(subject, *, jetstream=False, servers=None)` |
|
|
463
|
+
| `GooglePubSubSink` | `log_foundry.sinks.pubsub` | `gcp-pubsub` | `GooglePubSubSink(topic)` |
|
|
464
|
+
| `AzureEventHubsSink` | `log_foundry.sinks.eventhubs` | `azure-eventhubs` | `AzureEventHubsSink(*, connection_str="…", eventhub=None)` |
|
|
450
465
|
|
|
451
466
|
```python
|
|
452
|
-
from
|
|
467
|
+
from log_foundry.sinks.kafka import KafkaSink
|
|
453
468
|
lf.configure(sink=KafkaSink("app-logs", bootstrap_servers="broker:9092"))
|
|
454
469
|
```
|
|
455
470
|
|
|
@@ -459,9 +474,9 @@ Write-only inserts (querying is the downstream tool's job); each needs its own e
|
|
|
459
474
|
|
|
460
475
|
| Sink | Import from | Extra | Configure |
|
|
461
476
|
|---|---|---|---|
|
|
462
|
-
| `MongoDBSink` | `
|
|
463
|
-
| `PostgresSink` | `
|
|
464
|
-
| `ClickHouseSink` | `
|
|
477
|
+
| `MongoDBSink` | `log_foundry.sinks.mongodb` | `mongo` | `MongoDBSink(*, uri="…", database="…", collection="…")` |
|
|
478
|
+
| `PostgresSink` | `log_foundry.sinks.postgres` | `postgres` | `PostgresSink(table, *, dsn="…", create_table=False)` — JSONB `event` column + extracted columns |
|
|
479
|
+
| `ClickHouseSink` | `log_foundry.sinks.clickhouse` | `clickhouse` | `ClickHouseSink(table, *, dsn="…", create_table=False)` — MergeTree, columnar insert |
|
|
465
480
|
|
|
466
481
|
`PostgresSink` / `ClickHouseSink` default `create_table=False` (you own the schema and indexes); set
|
|
467
482
|
it `True` for an idempotent `CREATE TABLE IF NOT EXISTS` convenience.
|
|
@@ -481,7 +496,7 @@ them (`worker.dropped`) rather than stalling.
|
|
|
481
496
|
Because delivery is asynchronous, drain before the process exits:
|
|
482
497
|
|
|
483
498
|
```python
|
|
484
|
-
import
|
|
499
|
+
import log_foundry as lf
|
|
485
500
|
|
|
486
501
|
lf.shutdown() # flush buffered events and close the sink; blocks until drained
|
|
487
502
|
```
|
|
@@ -526,11 +541,41 @@ poetry run mypy # typecheck (strict, over src/)
|
|
|
526
541
|
every pull request and on push to `main`. A second workflow
|
|
527
542
|
([`spec-lint.yml`](.github/workflows/spec-lint.yml)) lints the design specs under `docs/specs/`.
|
|
528
543
|
|
|
529
|
-
The library uses a src layout (`src/
|
|
544
|
+
The library uses a src layout (`src/log_foundry/`) with a single concept per module: `config`,
|
|
530
545
|
`ids`, `model`, `context`, `decorator`, `api`, `console`, `worker`, and the `sinks/` package (the
|
|
531
546
|
`base` protocol, `stdout`, and one module per sink family — see [Sinks](#sinks)).
|
|
532
547
|
Deeper design docs live in [`docs/`](docs/) — start with [`docs/architecture.md`](docs/architecture.md).
|
|
533
548
|
|
|
549
|
+
## Releasing
|
|
550
|
+
|
|
551
|
+
**The version is never hand-edited.** It is derived from Git tags at build time by
|
|
552
|
+
`poetry-dynamic-versioning`, so `pyproject.toml` carries no literal version and the published
|
|
553
|
+
number can't drift from what Git says.
|
|
554
|
+
|
|
555
|
+
[`release.yml`](.github/workflows/release.yml) reuses the CI suite as a gate, then builds an
|
|
556
|
+
sdist and a wheel:
|
|
557
|
+
|
|
558
|
+
| Trigger | Version built | Published to PyPI as |
|
|
559
|
+
|---|---|---|
|
|
560
|
+
| merge to `main` | `X.Y.Z.devN` | dev pre-release |
|
|
561
|
+
| push tag `vX.Y.Z` | `X.Y.Z` | stable release |
|
|
562
|
+
|
|
563
|
+
Dev pre-releases keep the upload path exercised on every merge, so a real release is never the
|
|
564
|
+
first time it runs. `pip install log-foundry` still resolves to the latest **stable** version —
|
|
565
|
+
pip ignores pre-releases unless you pass `--pre`.
|
|
566
|
+
|
|
567
|
+
Cutting a release is one tag:
|
|
568
|
+
|
|
569
|
+
```bash
|
|
570
|
+
git tag -a v0.2.0 -m "log-foundry 0.2.0"
|
|
571
|
+
git push origin v0.2.0
|
|
572
|
+
```
|
|
573
|
+
|
|
574
|
+
Uploads authenticate with PyPI [Trusted Publishing](https://docs.pypi.org/trusted-publishers/)
|
|
575
|
+
(OIDC) through the `pypi` GitHub Environment — there is no API token stored in the repository.
|
|
576
|
+
A tagged build refuses to publish if the derived version doesn't match the tag, and the tagged
|
|
577
|
+
job deliberately omits `skip-existing` so re-pushing an already-published version fails loudly.
|
|
578
|
+
|
|
534
579
|
## License
|
|
535
580
|
|
|
536
581
|
[MIT](LICENSE) © Andrew Griffith
|