bigbrotr 5.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. bigbrotr-5.0.1/LICENSE +21 -0
  2. bigbrotr-5.0.1/PKG-INFO +545 -0
  3. bigbrotr-5.0.1/README.md +471 -0
  4. bigbrotr-5.0.1/pyproject.toml +257 -0
  5. bigbrotr-5.0.1/setup.cfg +4 -0
  6. bigbrotr-5.0.1/src/bigbrotr/__init__.py +79 -0
  7. bigbrotr-5.0.1/src/bigbrotr/__main__.py +246 -0
  8. bigbrotr-5.0.1/src/bigbrotr/core/__init__.py +82 -0
  9. bigbrotr-5.0.1/src/bigbrotr/core/base_service.py +296 -0
  10. bigbrotr-5.0.1/src/bigbrotr/core/brotr.py +814 -0
  11. bigbrotr-5.0.1/src/bigbrotr/core/exceptions.py +95 -0
  12. bigbrotr-5.0.1/src/bigbrotr/core/logger.py +209 -0
  13. bigbrotr-5.0.1/src/bigbrotr/core/metrics.py +179 -0
  14. bigbrotr-5.0.1/src/bigbrotr/core/pool.py +750 -0
  15. bigbrotr-5.0.1/src/bigbrotr/core/py.typed +0 -0
  16. bigbrotr-5.0.1/src/bigbrotr/core/yaml.py +41 -0
  17. bigbrotr-5.0.1/src/bigbrotr/models/__init__.py +46 -0
  18. bigbrotr-5.0.1/src/bigbrotr/models/constants.py +29 -0
  19. bigbrotr-5.0.1/src/bigbrotr/models/event.py +146 -0
  20. bigbrotr-5.0.1/src/bigbrotr/models/event_relay.py +122 -0
  21. bigbrotr-5.0.1/src/bigbrotr/models/metadata.py +308 -0
  22. bigbrotr-5.0.1/src/bigbrotr/models/py.typed +0 -0
  23. bigbrotr-5.0.1/src/bigbrotr/models/relay.py +305 -0
  24. bigbrotr-5.0.1/src/bigbrotr/models/relay_metadata.py +124 -0
  25. bigbrotr-5.0.1/src/bigbrotr/models/service_state.py +60 -0
  26. bigbrotr-5.0.1/src/bigbrotr/nips/__init__.py +13 -0
  27. bigbrotr-5.0.1/src/bigbrotr/nips/base.py +132 -0
  28. bigbrotr-5.0.1/src/bigbrotr/nips/nip11/__init__.py +54 -0
  29. bigbrotr-5.0.1/src/bigbrotr/nips/nip11/data.py +308 -0
  30. bigbrotr-5.0.1/src/bigbrotr/nips/nip11/fetch.py +227 -0
  31. bigbrotr-5.0.1/src/bigbrotr/nips/nip11/logs.py +20 -0
  32. bigbrotr-5.0.1/src/bigbrotr/nips/nip11/nip11.py +103 -0
  33. bigbrotr-5.0.1/src/bigbrotr/nips/nip66/__init__.py +95 -0
  34. bigbrotr-5.0.1/src/bigbrotr/nips/nip66/data.py +182 -0
  35. bigbrotr-5.0.1/src/bigbrotr/nips/nip66/dns.py +164 -0
  36. bigbrotr-5.0.1/src/bigbrotr/nips/nip66/geo.py +205 -0
  37. bigbrotr-5.0.1/src/bigbrotr/nips/nip66/http.py +158 -0
  38. bigbrotr-5.0.1/src/bigbrotr/nips/nip66/logs.py +110 -0
  39. bigbrotr-5.0.1/src/bigbrotr/nips/nip66/net.py +142 -0
  40. bigbrotr-5.0.1/src/bigbrotr/nips/nip66/nip66.py +270 -0
  41. bigbrotr-5.0.1/src/bigbrotr/nips/nip66/rtt.py +337 -0
  42. bigbrotr-5.0.1/src/bigbrotr/nips/nip66/ssl.py +285 -0
  43. bigbrotr-5.0.1/src/bigbrotr/nips/parsing.py +135 -0
  44. bigbrotr-5.0.1/src/bigbrotr/py.typed +0 -0
  45. bigbrotr-5.0.1/src/bigbrotr/services/__init__.py +69 -0
  46. bigbrotr-5.0.1/src/bigbrotr/services/common/__init__.py +74 -0
  47. bigbrotr-5.0.1/src/bigbrotr/services/common/configs.py +175 -0
  48. bigbrotr-5.0.1/src/bigbrotr/services/common/constants.py +39 -0
  49. bigbrotr-5.0.1/src/bigbrotr/services/common/mixins.py +135 -0
  50. bigbrotr-5.0.1/src/bigbrotr/services/common/queries.py +468 -0
  51. bigbrotr-5.0.1/src/bigbrotr/services/finder.py +493 -0
  52. bigbrotr-5.0.1/src/bigbrotr/services/monitor.py +1004 -0
  53. bigbrotr-5.0.1/src/bigbrotr/services/monitor_publisher.py +231 -0
  54. bigbrotr-5.0.1/src/bigbrotr/services/monitor_tags.py +282 -0
  55. bigbrotr-5.0.1/src/bigbrotr/services/py.typed +0 -0
  56. bigbrotr-5.0.1/src/bigbrotr/services/seeder.py +194 -0
  57. bigbrotr-5.0.1/src/bigbrotr/services/synchronizer.py +727 -0
  58. bigbrotr-5.0.1/src/bigbrotr/services/validator.py +473 -0
  59. bigbrotr-5.0.1/src/bigbrotr/utils/__init__.py +31 -0
  60. bigbrotr-5.0.1/src/bigbrotr/utils/dns.py +54 -0
  61. bigbrotr-5.0.1/src/bigbrotr/utils/keys.py +89 -0
  62. bigbrotr-5.0.1/src/bigbrotr/utils/py.typed +0 -0
  63. bigbrotr-5.0.1/src/bigbrotr/utils/transport.py +520 -0
  64. bigbrotr-5.0.1/src/bigbrotr.egg-info/PKG-INFO +545 -0
  65. bigbrotr-5.0.1/src/bigbrotr.egg-info/SOURCES.txt +67 -0
  66. bigbrotr-5.0.1/src/bigbrotr.egg-info/dependency_links.txt +1 -0
  67. bigbrotr-5.0.1/src/bigbrotr.egg-info/entry_points.txt +2 -0
  68. bigbrotr-5.0.1/src/bigbrotr.egg-info/requires.txt +42 -0
  69. bigbrotr-5.0.1/src/bigbrotr.egg-info/top_level.txt +1 -0
bigbrotr-5.0.1/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 BigBrotr
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,545 @@
1
+ Metadata-Version: 2.4
2
+ Name: bigbrotr
3
+ Version: 5.0.1
4
+ Summary: A modular Nostr data archiving and monitoring system
5
+ Author: BigBrotr Contributors
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/BigBrotr/bigbrotr
8
+ Project-URL: Repository, https://github.com/BigBrotr/bigbrotr
9
+ Project-URL: Documentation, https://bigbrotr.github.io/bigbrotr/
10
+ Project-URL: Issues, https://github.com/BigBrotr/bigbrotr/issues
11
+ Project-URL: Changelog, https://github.com/BigBrotr/bigbrotr/blob/main/CHANGELOG.md
12
+ Keywords: nostr,archiving,monitoring,postgresql,async,relay,nip-11,nip-66
13
+ Classifier: Development Status :: 4 - Beta
14
+ Classifier: Environment :: Console
15
+ Classifier: Intended Audience :: Developers
16
+ Classifier: Intended Audience :: System Administrators
17
+ Classifier: License :: OSI Approved :: MIT License
18
+ Classifier: Operating System :: OS Independent
19
+ Classifier: Programming Language :: Python :: 3
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Programming Language :: Python :: 3.13
23
+ Classifier: Programming Language :: Python :: 3.14
24
+ Classifier: Topic :: Database
25
+ Classifier: Topic :: Internet
26
+ Classifier: Topic :: System :: Archiving
27
+ Classifier: Topic :: System :: Monitoring
28
+ Classifier: Framework :: AsyncIO
29
+ Classifier: Typing :: Typed
30
+ Requires-Python: >=3.11
31
+ Description-Content-Type: text/markdown
32
+ License-File: LICENSE
33
+ Requires-Dist: aiohttp>=3.9.0
34
+ Requires-Dist: aiohttp-socks>=0.9.0
35
+ Requires-Dist: aiomultiprocess>=0.9.1
36
+ Requires-Dist: asyncpg>=0.29.0
37
+ Requires-Dist: dnspython>=2.5.0
38
+ Requires-Dist: geoip2>=4.8.0
39
+ Requires-Dist: geohash2>=1.1
40
+ Requires-Dist: Jinja2>=3.1.0
41
+ Requires-Dist: nostr-sdk>=0.37.0
42
+ Requires-Dist: prometheus-client>=0.20.0
43
+ Requires-Dist: pydantic>=2.5.0
44
+ Requires-Dist: PyYAML>=6.0.1
45
+ Requires-Dist: rfc3986>=2.0.0
46
+ Requires-Dist: tldextract>=5.1.0
47
+ Requires-Dist: typing-extensions>=4.9.0
48
+ Provides-Extra: dev
49
+ Requires-Dist: pytest>=8.0.0; extra == "dev"
50
+ Requires-Dist: pytest-asyncio>=0.24.0; extra == "dev"
51
+ Requires-Dist: pytest-cov>=5.0.0; extra == "dev"
52
+ Requires-Dist: pytest-mock>=3.14.0; extra == "dev"
53
+ Requires-Dist: pytest-timeout>=2.3.0; extra == "dev"
54
+ Requires-Dist: pytest-xdist>=3.5.0; extra == "dev"
55
+ Requires-Dist: ruff>=0.14.0; extra == "dev"
56
+ Requires-Dist: mypy>=1.19.0; extra == "dev"
57
+ Requires-Dist: yamllint>=1.37.0; extra == "dev"
58
+ Requires-Dist: sqlfluff>=3.5.0; extra == "dev"
59
+ Requires-Dist: pre-commit>=4.0.0; extra == "dev"
60
+ Requires-Dist: detect-secrets>=1.5.0; extra == "dev"
61
+ Requires-Dist: pip-audit>=2.7.0; extra == "dev"
62
+ Requires-Dist: build>=1.0.0; extra == "dev"
63
+ Requires-Dist: twine>=6.0.0; extra == "dev"
64
+ Requires-Dist: types-PyYAML>=6.0.12; extra == "dev"
65
+ Requires-Dist: asyncpg-stubs>=0.30.0; extra == "dev"
66
+ Requires-Dist: testcontainers[postgres]>=4.0.0; extra == "dev"
67
+ Requires-Dist: alembic>=1.14.0; extra == "dev"
68
+ Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == "dev"
69
+ Provides-Extra: docs
70
+ Requires-Dist: mkdocs>=1.6.0; extra == "docs"
71
+ Requires-Dist: mkdocs-material>=9.5.0; extra == "docs"
72
+ Requires-Dist: mkdocstrings[python]>=0.27.0; extra == "docs"
73
+ Dynamic: license-file
74
+
75
+ <p align="center">
76
+ <img src="https://img.shields.io/badge/python-3.11+-3776AB?style=for-the-badge&logo=python&logoColor=white" alt="Python 3.11+">
77
+ <img src="https://img.shields.io/badge/postgresql-16+-4169E1?style=for-the-badge&logo=postgresql&logoColor=white" alt="PostgreSQL 16+">
78
+ <img src="https://img.shields.io/badge/async-asyncpg-00ADD8?style=for-the-badge" alt="Async">
79
+ <img src="https://img.shields.io/badge/docker-compose-2496ED?style=for-the-badge&logo=docker&logoColor=white" alt="Docker">
80
+ <img src="https://img.shields.io/badge/license-MIT-green?style=for-the-badge" alt="MIT License">
81
+ <a href="https://codecov.io/gh/Bigbrotr/bigbrotr"><img src="https://img.shields.io/codecov/c/github/Bigbrotr/bigbrotr?token=LM9D3ABW0L&style=for-the-badge&logo=codecov&logoColor=white&label=coverage" alt="Coverage"></a>
82
+ </p>
83
+
84
+ <h1 align="center">BigBrotr</h1>
85
+
86
+ <p align="center">
87
+ <strong>Nostr Relay Discovery, Monitoring, and Event Archiving System</strong>
88
+ </p>
89
+
90
+ <p align="center">
91
+ Discovers relays across clearnet and overlay networks, monitors health with NIP-11/NIP-66 compliance checks, and archives events into PostgreSQL.
92
+ </p>
93
+
94
+ ---
95
+
96
+ ## What It Does
97
+
98
+ BigBrotr runs a pipeline of five async services that continuously map and monitor the Nostr relay ecosystem:
99
+
100
+ ```text
101
+ Seeder ──> Finder ──> Validator ──> Monitor ──> Synchronizer
102
+ (seed URLs) (discover) (test) (health) (archive events)
103
+ ```
104
+
105
+ 1. **Seeder** loads relay URLs from a seed file (one-shot)
106
+ 2. **Finder** discovers new relays from stored events (NIP-65 relay lists, kind 2/3) and external APIs
107
+ 3. **Validator** tests WebSocket connectivity for each candidate, promoting valid relays
108
+ 4. **Monitor** performs NIP-11 info document fetches and NIP-66 health checks (RTT, SSL, DNS, GeoIP, ASN, HTTP headers), then publishes results as kind 10166/30166 Nostr events
109
+ 5. **Synchronizer** connects to all validated relays, subscribes to events, and archives them with per-relay cursor tracking for incremental sync
110
+
111
+ All services expose Prometheus metrics, run behind PGBouncer connection pooling, and support clearnet + Tor + I2P + Lokinet connectivity.
112
+
113
+ ---
114
+
115
+ ## Architecture
116
+
117
+ Diamond DAG with strict import direction (top to bottom only):
118
+
119
+ ```text
120
+ services src/bigbrotr/services/
121
+ / | \
122
+ core nips utils src/bigbrotr/{core,nips,utils}/
123
+ \ | /
124
+ models src/bigbrotr/models/
125
+ ```
126
+
127
+ - **models** -- Pure frozen dataclasses. Zero I/O, zero package dependencies, stdlib logging only.
128
+ - **core** -- Pool (asyncpg), Brotr (DB facade), BaseService, Logger, Metrics, Exceptions.
129
+ - **nips** -- NIP-11 relay info fetch/parse, NIP-66 health checks (RTT, SSL, DNS, Geo, Net, HTTP). Has I/O.
130
+ - **utils** -- DNS resolution, Nostr key management, WebSocket/HTTP transport with SOCKS5 proxy.
131
+ - **services** -- Business logic: Seeder, Finder, Validator, Monitor (+ Publisher + Tags), Synchronizer.
132
+
133
+ ---
134
+
135
+ ## Quick Start
136
+
137
+ ### Prerequisites
138
+
139
+ - Docker and Docker Compose
140
+ - (Optional) Python 3.11+ for local development
141
+
142
+ ### Deploy with Docker Compose
143
+
144
+ ```bash
145
+ git clone https://github.com/BigBrotr/bigbrotr.git
146
+ cd bigbrotr/deployments/bigbrotr
147
+
148
+ # Configure secrets
149
+ cp .env.example .env
150
+ # Edit .env: set DB_PASSWORD, PRIVATE_KEY, GRAFANA_PASSWORD
151
+
152
+ # Start everything
153
+ docker compose up -d
154
+
155
+ # Watch the pipeline start
156
+ docker compose logs -f seeder
157
+ ```
158
+
159
+ This starts PostgreSQL, PGBouncer, Tor proxy, all 5 services, Prometheus, and Grafana.
160
+
161
+ | Endpoint | URL |
162
+ |----------|-----|
163
+ | Grafana | `http://localhost:3000` |
164
+ | Prometheus | `http://localhost:9090` |
165
+ | PostgreSQL | `localhost:5432` |
166
+ | PGBouncer | `localhost:6432` |
167
+
168
+ ### Run a Single Service Locally
169
+
170
+ ```bash
171
+ pip install -e ".[dev]"
172
+ cd deployments/bigbrotr
173
+
174
+ # One cycle
175
+ python -m bigbrotr seeder --once
176
+
177
+ # Continuous with debug logging
178
+ python -m bigbrotr finder --log-level DEBUG
179
+ ```
180
+
181
+ ---
182
+
183
+ ## Deployments
184
+
185
+ BigBrotr supports multiple deployment configurations from the same codebase via a single parametric Dockerfile (`deployments/Dockerfile` with `ARG DEPLOYMENT`).
186
+
187
+ ### BigBrotr (Full Archive)
188
+
189
+ Stores complete Nostr events (id, pubkey, created_at, kind, tags, content, sig). 7 materialized views for analytics. Tor enabled. All 5 services + Prometheus + Grafana.
190
+
191
+ ```bash
192
+ cd deployments/bigbrotr && docker compose up -d
193
+ ```
194
+
195
+ ### LilBrotr (Lightweight)
196
+
197
+ Stores event metadata only (id, pubkey, created_at, kind, tagvalues). Omits tags JSON, content, and sig for approximately 60% disk savings. No materialized views. Same service pipeline.
198
+
199
+ ```bash
200
+ cd deployments/lilbrotr && docker compose up -d
201
+ ```
202
+
203
+ ### Custom Deployment
204
+
205
+ ```bash
206
+ cp -r deployments/_template deployments/myrelay
207
+ # Edit config, SQL schema, docker-compose.yaml
208
+ cd deployments/myrelay && docker compose up -d
209
+ ```
210
+
211
+ ---
212
+
213
+ ## Database
214
+
215
+ PostgreSQL 16 with PGBouncer (transaction-mode pooling) and asyncpg async driver. All mutations via stored functions with bulk array parameters.
216
+
217
+ ### Schema
218
+
219
+ | Table | Purpose |
220
+ |-------|---------|
221
+ | `relay` | Validated relay URLs with network type and discovery timestamp |
222
+ | `event` | Nostr events (BYTEA ids/pubkeys/sigs for space efficiency) |
223
+ | `event_relay` | Junction: which events were seen at which relays |
224
+ | `metadata` | Content-addressed NIP-11/NIP-66 documents (SHA-256 dedup, `payload` JSONB) |
225
+ | `relay_metadata` | Time-series snapshots linking relays to metadata records (`metadata_type` column) |
226
+ | `service_state` | Per-service operational data (candidates, cursors, checkpoints) |
227
+
228
+ ### Stored Functions (22)
229
+
230
+ - **1 utility**: `tags_to_tagvalues` (extracts single-char tag values for GIN indexing)
231
+ - **10 CRUD**: `relay_insert`, `event_insert`, `metadata_insert`, `event_relay_insert`, `relay_metadata_insert`, `event_relay_insert_cascade`, `relay_metadata_insert_cascade`, `service_state_upsert`, `service_state_get`, `service_state_delete`
232
+ - **3 cleanup**: `orphan_event_delete`, `orphan_metadata_delete`, `relay_metadata_delete_expired` (all batched)
233
+ - **8 refresh**: one per materialized view + `all_statistics_refresh`
234
+
235
+ All functions use `SECURITY INVOKER`, bulk array parameters, and `ON CONFLICT DO NOTHING`.
236
+
237
+ ### Materialized Views (7, BigBrotr Only)
238
+
239
+ `relay_metadata_latest`, `event_stats`, `relay_stats`, `kind_counts`, `kind_counts_by_relay`, `pubkey_counts`, `pubkey_counts_by_relay` -- all support `REFRESH CONCURRENTLY` via unique indexes.
240
+
241
+ ---
242
+
243
+ ## Monitoring
244
+
245
+ ### Prometheus Metrics
246
+
247
+ Every service exposes `/metrics` on its configured port with four metric types:
248
+
249
+ | Metric | Type | Description |
250
+ |--------|------|-------------|
251
+ | `service_info` | Info | Static service metadata |
252
+ | `service_gauge` | Gauge | Point-in-time state (consecutive_failures, last_cycle_timestamp, progress) |
253
+ | `service_counter` | Counter | Cumulative totals (cycles_success, cycles_failed, errors by type) |
254
+ | `cycle_duration_seconds` | Histogram | Cycle latency with 10 buckets (1s to 1h) |
255
+
256
+ ### Alert Rules (4)
257
+
258
+ | Alert | Condition | Severity |
259
+ |-------|-----------|----------|
260
+ | ServiceDown | `up == 0` for 5m | critical |
261
+ | HighFailureRate | error rate > 0.1/s for 5m | warning |
262
+ | PoolExhausted | zero available connections for 2m | critical |
263
+ | DatabaseSlow | p99 query latency > 5s for 5m | warning |
264
+
265
+ ### Grafana Dashboard
266
+
267
+ Auto-provisioned dashboard with per-service panels: last cycle time, cycle duration, error counts (24h), consecutive failures. Validator has additional candidate progress panels.
268
+
269
+ ### Structured Logging
270
+
271
+ ```text
272
+ info finder cycle_completed relay_count=100 duration=2.5
273
+ error validator retry_failed attempt=3 url="wss://relay.example.com"
274
+ ```
275
+
276
+ JSON mode available for cloud aggregation:
277
+
278
+ ```json
279
+ {"timestamp": "2026-02-09T12:34:56+00:00", "level": "info", "service": "finder", "message": "cycle_completed", "relay_count": 100}
280
+ ```
281
+
282
+ ---
283
+
284
+ ## Nostr Protocol Support
285
+
286
+ ### NIPs Implemented
287
+
288
+ | NIP | Usage |
289
+ |-----|-------|
290
+ | **NIP-01** | Event model, relay communication |
291
+ | **NIP-02** | Contact list relay discovery (kind 3) |
292
+ | **NIP-11** | Relay information document fetch and parse |
293
+ | **NIP-65** | Relay list metadata (kind 10002) |
294
+ | **NIP-66** | Relay monitoring and discovery (kinds 10166, 30166) |
295
+
296
+ ### Event Kinds
297
+
298
+ | Kind | Direction | Purpose |
299
+ |------|-----------|---------|
300
+ | 0 | Published | Monitor profile metadata |
301
+ | 2 | Consumed | Deprecated relay recommendation (content = URL) |
302
+ | 3 | Consumed | Contact list (content = JSON with relay URLs as keys) |
303
+ | 10002 | Consumed | NIP-65 relay list ("r" tags) |
304
+ | 10166 | Published | Monitor announcement (capabilities, timeouts) |
305
+ | 30166 | Published | Relay discovery (addressable, one per relay, full metadata tags) |
306
+
307
+ ### NIP-66 Health Checks
308
+
309
+ | Check | What It Measures |
310
+ |-------|-----------------|
311
+ | RTT | WebSocket open/read/write latency (ms) |
312
+ | SSL | Certificate validity, expiry, issuer, cipher suite |
313
+ | DNS | A/AAAA/CNAME/NS/PTR records, query time |
314
+ | Geo | Country, city, coordinates, timezone, geohash (GeoLite2) |
315
+ | Net | IP address, ASN, organization (GeoLite2 ASN) |
316
+ | HTTP | Server header, X-Powered-By |
317
+
318
+ ---
319
+
320
+ ## Configuration
321
+
322
+ ### Environment Variables
323
+
324
+ | Variable | Required | Description |
325
+ |----------|----------|-------------|
326
+ | `DB_PASSWORD` | Yes | PostgreSQL password |
327
+ | `PRIVATE_KEY` | For Monitor | Nostr private key (hex or nsec) for event publishing and RTT write tests |
328
+ | `GRAFANA_PASSWORD` | No | Grafana admin password |
329
+
330
+ ### Configuration Files
331
+
332
+ ```text
333
+ deployments/bigbrotr/config/
334
+ +-- brotr.yaml # Pool, batch size, timeouts
335
+ +-- services/
336
+ +-- seeder.yaml # Seed file path
337
+ +-- finder.yaml # API sources, scan interval (default: 1h)
338
+ +-- validator.yaml # Validation interval (8h), cleanup, networks
339
+ +-- monitor.yaml # Check interval (1h), retry per check type, networks
340
+ +-- synchronizer.yaml # Sync interval (15m), per-relay overrides, concurrency
341
+ ```
342
+
343
+ All configs use Pydantic v2 validation with typed defaults and constraints.
344
+
345
+ ---
346
+
347
+ ## Development
348
+
349
+ ### Setup
350
+
351
+ ```bash
352
+ git clone https://github.com/BigBrotr/bigbrotr.git && cd bigbrotr
353
+ python3 -m venv .venv && source .venv/bin/activate
354
+ pip install -e ".[dev]"
355
+ pre-commit install
356
+ ```
357
+
358
+ ### Quality Checks
359
+
360
+ ```bash
361
+ make lint # ruff check src/ tests/
362
+ make format # ruff format src/ tests/
363
+ make typecheck # mypy src/bigbrotr (strict mode)
364
+ make test-unit # pytest unit tests (2049 tests)
365
+ make test-integration # pytest integration tests (requires Docker)
366
+ make test-fast # pytest -m "not slow"
367
+ make coverage # pytest --cov with HTML report
368
+ make ci # all checks: lint + format + typecheck + test-unit
369
+ make docs # build MkDocs documentation site
370
+ make docs-serve # serve docs locally with live reload
371
+ make build # build Python package (sdist + wheel)
372
+ make docker-build # build Docker image (DEPLOYMENT=bigbrotr)
373
+ make docker-up # start Docker stack
374
+ make docker-down # stop Docker stack
375
+ make clean # remove build artifacts and caches
376
+ ```
377
+
378
+ ### Test Suite
379
+
380
+ - **2049 unit tests** + **8 integration tests** (testcontainers PostgreSQL)
381
+ - `asyncio_mode = "auto"` -- no `@pytest.mark.asyncio` needed
382
+ - Global timeout: 120s per test
383
+ - Shared fixtures via `tests/fixtures/relays.py` (registered as pytest plugin)
384
+ - Coverage threshold: 80% (branch coverage enabled)
385
+
386
+ ### CI/CD Pipeline
387
+
388
+ | Stage | Tool | Purpose |
389
+ |-------|------|---------|
390
+ | Pre-commit | ruff, mypy, yamllint, detect-secrets, markdownlint, hadolint, sqlfluff | Code quality gates |
391
+ | Unit Test | pytest (Python 3.11--3.14 matrix) | Unit tests + coverage |
392
+ | Integration Test | pytest + testcontainers | PostgreSQL integration tests |
393
+ | Build | Docker Buildx (matrix) | Multi-deployment image builds + Trivy scan |
394
+ | Security | pip-audit, Trivy, CodeQL | Dependency vulns, container scanning, static analysis |
395
+ | Release | PyPI (OIDC) + GHCR | Package + Docker image publishing, SBOM generation |
396
+ | Docs | MkDocs Material | Auto-generated API docs deployed to GitHub Pages |
397
+ | Dependencies | Dependabot | Weekly updates for pip, Docker, GitHub Actions |
398
+
399
+ ---
400
+
401
+ ## Project Structure
402
+
403
+ ```text
404
+ bigbrotr/
405
+ +-- src/bigbrotr/ # Main package (namespace)
406
+ | +-- __main__.py # CLI entry point
407
+ | +-- core/ # Foundation
408
+ | | +-- pool.py # asyncpg connection pool with retry
409
+ | | +-- brotr.py # High-level DB facade (stored procedures)
410
+ | | +-- base_service.py # Abstract service with run_forever loop
411
+ | | +-- exceptions.py # Exception hierarchy (10 classes)
412
+ | | +-- metrics.py # Prometheus metrics server
413
+ | | +-- logger.py # Structured key=value / JSON logging
414
+ | | +-- yaml.py # YAML config loader
415
+ | +-- models/ # Pure frozen dataclasses (zero I/O)
416
+ | | +-- relay.py # URL validation, network detection
417
+ | | +-- event.py # Nostr event wrapper
418
+ | | +-- metadata.py # Content-addressed metadata (SHA-256)
419
+ | | +-- service_state.py # ServiceState, EventKind, StateType
420
+ | | +-- constants.py # NetworkType enum
421
+ | | +-- event_relay.py # Event-relay junction
422
+ | | +-- relay_metadata.py # Relay-metadata junction
423
+ | +-- nips/ # NIP protocol implementations (I/O)
424
+ | | +-- nip11/ # Relay information document
425
+ | | +-- nip66/ # Health checks: rtt, ssl, dns, geo, net, http
426
+ | +-- utils/ # DNS, keys, transport
427
+ | +-- services/ # Business logic
428
+ | +-- seeder.py
429
+ | +-- finder.py
430
+ | +-- validator.py
431
+ | +-- monitor.py # Health check orchestration
432
+ | +-- monitor_publisher.py # Nostr event broadcasting
433
+ | +-- monitor_tags.py # NIP-66 tag building
434
+ | +-- synchronizer.py
435
+ | +-- common/ # Shared queries, configs, mixins
436
+ +-- deployments/
437
+ | +-- Dockerfile # Single parametric (ARG DEPLOYMENT)
438
+ | +-- bigbrotr/ # Full archive deployment
439
+ | | +-- config/ # YAML configs
440
+ | | +-- postgres/init/ # SQL schema (10 files, 22 functions)
441
+ | | +-- monitoring/ # Prometheus + Grafana provisioning
442
+ | | +-- docker-compose.yaml
443
+ | +-- lilbrotr/ # Lightweight deployment
444
+ | +-- _template/ # Custom deployment template
445
+ +-- tests/
446
+ | +-- fixtures/relays.py # Shared relay fixtures
447
+ | +-- unit/ # 2049 tests (mirrors src/ structure)
448
+ | +-- integration/ # 8 tests (testcontainers PostgreSQL)
449
+ +-- docs/ # Architecture, Database, Deployment, Development, Configuration
450
+ +-- Makefile # Development targets
451
+ +-- pyproject.toml # All config: deps, ruff, mypy, pytest, coverage
452
+ ```
453
+
454
+ ---
455
+
456
+ ## Exception Hierarchy
457
+
458
+ ```text
459
+ BigBrotrError
460
+ +-- ConfigurationError # YAML, env vars, CLI
461
+ +-- DatabaseError
462
+ | +-- ConnectionPoolError # Transient (retry)
463
+ | +-- QueryError # Permanent (don't retry)
464
+ +-- ConnectivityError
465
+ | +-- RelayTimeoutError # Connection/response timed out
466
+ | +-- RelaySSLError # TLS/SSL failures
467
+ +-- ProtocolError # NIP parsing/validation
468
+ +-- PublishingError # Event broadcast failures
469
+ ```
470
+
471
+ ---
472
+
473
+ ## Docker Infrastructure
474
+
475
+ ### Container Stack
476
+
477
+ | Container | Image | Purpose | Resources |
478
+ |-----------|-------|---------|-----------|
479
+ | postgres | `postgres:16-alpine` | Primary storage | 2 CPU, 2 GB |
480
+ | pgbouncer | `edoburu/pgbouncer:v1.25.1-p0` | Transaction-mode connection pooling | 0.5 CPU, 256 MB |
481
+ | tor | `osminogin/tor-simple:0.4.8.10` | SOCKS5 proxy for .onion relays | 0.5 CPU, 256 MB |
482
+ | finder | bigbrotr (parametric) | Relay discovery | 1 CPU, 512 MB |
483
+ | validator | bigbrotr (parametric) | Candidate validation | 1 CPU, 512 MB |
484
+ | monitor | bigbrotr (parametric) | Health monitoring | 1 CPU, 512 MB |
485
+ | synchronizer | bigbrotr (parametric) | Event archiving | 1 CPU, 512 MB |
486
+ | prometheus | `prom/prometheus:v2.51.0` | Metrics collection (30d retention) | 0.5 CPU, 512 MB |
487
+ | grafana | `grafana/grafana:10.4.1` | Dashboards | 0.5 CPU, 512 MB |
488
+
489
+ ### Networks
490
+
491
+ - `data-network` -- postgres, pgbouncer, tor, all services
492
+ - `monitoring-network` -- prometheus, grafana, all services (metrics scraping)
493
+
494
+ ### Security
495
+
496
+ - All ports bound to `127.0.0.1` (no external exposure)
497
+ - Non-root container execution (UID 1000)
498
+ - `tini` as PID 1 for proper signal handling
499
+ - SCRAM-SHA-256 authentication (PostgreSQL + PGBouncer)
500
+ - Real healthchecks via `/metrics` endpoint (not fake PID checks)
501
+
502
+ ---
503
+
504
+ ## Technology Stack
505
+
506
+ | Category | Technologies |
507
+ |----------|-------------|
508
+ | Language | Python 3.11+ (fully typed, strict mypy) |
509
+ | Database | PostgreSQL 16, asyncpg, PGBouncer |
510
+ | Async | asyncio, aiohttp, aiomultiprocess |
511
+ | Nostr | nostr-sdk (Rust FFI via PyO3/uniffi) |
512
+ | Validation | Pydantic v2, rfc3986 |
513
+ | Monitoring | Prometheus, Grafana, structured logging |
514
+ | Networking | aiohttp-socks (SOCKS5), dnspython, geoip2, tldextract |
515
+ | Testing | pytest, pytest-asyncio, pytest-cov, testcontainers |
516
+ | Quality | ruff (lint+format), mypy (strict), pre-commit (21 hooks) |
517
+ | CI/CD | GitHub Actions, pip-audit, Trivy, CodeQL, Dependabot |
518
+ | Containers | Docker, Docker Compose, tini |
519
+
520
+ ---
521
+
522
+ ## Contributing
523
+
524
+ 1. Fork and clone
525
+ 2. `pip install -e ".[dev]"` and `pre-commit install`
526
+ 3. Write tests for new functionality
527
+ 4. `make ci` -- all checks must pass
528
+ 5. Submit a pull request
529
+
530
+ Conventional commits: `feat:`, `fix:`, `refactor:`, `docs:`, `test:`, `chore:`
531
+
532
+ ---
533
+
534
+ ## License
535
+
536
+ MIT -- see [LICENSE](LICENSE).
537
+
538
+ ---
539
+
540
+ ## Links
541
+
542
+ - [Nostr Protocol](https://nostr.com)
543
+ - [NIP-11: Relay Information Document](https://github.com/nostr-protocol/nips/blob/master/11.md)
544
+ - [NIP-66: Relay Discovery and Monitoring](https://github.com/nostr-protocol/nips/blob/master/66.md)
545
+ - [NIPs Repository](https://github.com/nostr-protocol/nips)