lumonox 0.2.5__tar.gz

lumonox-0.2.5/.env.example

@@ -0,0 +1,126 @@
+# Copy to backend/.env and fill in values (never commit backend/.env).
+# Synthetic stack: scripts/run_synthetic_stack.sh sources this file before
+# `npm --prefix frontend run build`, so set NEXT_PUBLIC_* here first.
+#
+# Local / uvicorn: the backend loads this file automatically when it exists
+# next to this package (repo ``backend/.env``) or set LUMONOX_BACKEND_DOTENV to an absolute path.
+
+# Relational DB (projects, keys, dashboard metadata). Path is anchored to repo root / LUMONOX_DATA_DIR
+# (see normalize_database_url); keep alongside DuckDB under .lumonox/.
+DATABASE_URL=sqlite+aiosqlite:///./.lumonox/lumonox.db
+# Multi-replica production: run migrations once as a deploy step, then set to false on API replicas.
+# DATABASE_RUN_MIGRATIONS_ON_STARTUP=false
+
+# Raw HTTP events (local / synthetic stack)
+LUMONOX_EVENT_STORE=duckdb
+LUMONOX_EVENT_PLANE_MODE=duckdb_single_writer
+# Relative paths are anchored to the repo root (parent of backend/), not process cwd.
+# Prefer an absolute path in production. Override anchor with LUMONOX_DATA_DIR if needed.
+# Relative paths anchor to data root (LUMONOX_DATA_DIR / LUMONOX_PROJECT_ROOT, else monorepo root).
+# Prefer an absolute path in production. See resolve_lumonox_data_root in core/config.py.
+LUMONOX_DUCKDB_PATH=.lumonox/events.duckdb
+# Plan B (duckdb_log_shards): append-only shard storage root.
+# Relative paths are anchored to data root (same as LUMONOX_DUCKDB_PATH).
+LUMONOX_EVENT_PLANE_SHARDS_PATH=.lumonox/events-log
+# Plan B compactor snapshot output root.
+LUMONOX_EVENT_PLANE_SNAPSHOTS_PATH=.lumonox/events-duckdb
+# Optional phase-1 cold export (DuckDB -> partitioned Parquet):
+# LUMONOX_PARQUET_EXPORT_ENABLED=true
+# LUMONOX_PARQUET_EXPORT_ROOT=.lumonox/parquet/events
+# LUMONOX_PARQUET_EXPORT_INTERVAL_SECONDS=300
+# LUMONOX_PARQUET_EXPORT_WINDOW_SECONDS=900
+# Optional phase-2 hybrid read path (DuckDB hot + Parquet cold):
+# LUMONOX_PARQUET_QUERY_ENABLED=true
+# LUMONOX_PARQUET_HOT_WINDOW_HOURS=24
+# Optional phase-3 lifecycle (compaction + retention + readability verification):
+# LUMONOX_PARQUET_LIFECYCLE_ENABLED=true
+# LUMONOX_PARQUET_LIFECYCLE_INTERVAL_SECONDS=3600
+# LUMONOX_PARQUET_LIFECYCLE_RETENTION_DAYS=90
+# LUMONOX_PARQUET_LIFECYCLE_DRY_RUN=false
+# LUMONOX_PARQUET_LIFECYCLE_COMPACTION_MIN_FILES=4
+# LUMONOX_PARQUET_LIFECYCLE_VERIFY_SAMPLE_SIZE=5
+# Optional phase-4 object storage snapshots + DR restore:
+# LUMONOX_PARQUET_OBJECT_STORAGE_ENABLED=true
+# s3:// URIs need boto3 (Dockerfile installs backend[parquet-s3]; locally: pip install -e "./backend[parquet-s3]").
+# LUMONOX_PARQUET_OBJECT_STORAGE_URI=s3://my-bucket/lumonox
+# LUMONOX_PARQUET_OBJECT_STORAGE_PREFIX=parquet-events
+# LUMONOX_PARQUET_OBJECT_STORAGE_INTERVAL_SECONDS=900
+# LUMONOX_PARQUET_OBJECT_STORAGE_VERIFY_UPLOAD=true
+# LUMONOX_PARQUET_OBJECT_STORAGE_ENDPOINT_URL=https://s3.us-east-1.amazonaws.com
+# LUMONOX_PARQUET_OBJECT_STORAGE_REGION=us-east-1
+# LUMONOX_PARQUET_OBJECT_STORAGE_ACCESS_KEY_ID=
+# LUMONOX_PARQUET_OBJECT_STORAGE_SECRET_ACCESS_KEY=
+# LUMONOX_PARQUET_OBJECT_STORAGE_SESSION_TOKEN=
+# LUMONOX_PARQUET_OBJECT_STORAGE_RESTORE_ROOT=.lumonox/parquet/restore
+# LUMONOX_PARQUET_OBJECT_STORAGE_RESTORE_MANIFEST_KEY=
+# Optional Plan B knobs (kept conservative by default):
+# LUMONOX_SHARD_MAX_BYTES=134217728
+# LUMONOX_SHARD_MAX_AGE_SECONDS=300
+# LUMONOX_COMPACTOR_INTERVAL_SECONDS=60
+# LUMONOX_COMPACTOR_MAX_CONCURRENCY=1
+# LUMONOX_COMPACTOR_MAX_SHARDS_PER_RUN=1024
+# LUMONOX_COMPACTOR_PUBLISH_TIMEOUT_SECONDS=60
+# LUMONOX_SNAPSHOT_RETENTION_COUNT=3
+# LUMONOX_EVENT_PLANE_BACKPRESSURE_MIN_FREE_BYTES=536870912
+# LUMONOX_EVENT_PLANE_BACKPRESSURE_MIN_FREE_PERCENT=5
+# LUMONOX_EVENT_PLANE_BACKPRESSURE_MAX_PENDING_SHARDS=20000
+# LUMONOX_DATA_DIR=/srv/lumonox
+
+# Scheduler defaults:
+# - Local default SQLite filename (.lumonox/lumonox.db) auto-enables scheduler when unset.
+# - Non-default SQLite/Postgres should set JOBS_ENABLE_SCHEDULER=true explicitly (or run external cron jobs).
+# JOBS_ENABLE_SCHEDULER=true
+
+# Production topology baseline (recommended explicit values):
+# LUMONOX_ENV=production
+# LUMONOX_EVENT_STORE=duckdb
+# LUMONOX_EVENT_PLANE_MODE=duckdb_single_writer
+# LUMONOX_DUCKDB_SINGLE_WRITER_PROFILE=true   # required ack for single-writer DuckDB in production
+# JOBS_ENABLE_SCHEDULER=true
+# DASHBOARD_AUTH_ENABLED=true
+# DASHBOARD_ENFORCE_ORIGIN_FOR_MUTATIONS=true
+# DASHBOARD_REALTIME_BUS_BACKEND=postgres_notify  # set "none" for single-replica + sticky WS
+# DATABASE_RUN_MIGRATIONS_ON_STARTUP=false        # run one-shot migration before scaling replicas
+
+# Dashboard read/query protection for expensive endpoints (set 0 to disable):
+# DASHBOARD_READ_RATE_LIMIT_REQUESTS_PER_WINDOW=120
+# DASHBOARD_READ_RATE_LIMIT_WINDOW_SECONDS=60
+# Optional cross-replica WS propagation on Postgres:
+# DASHBOARD_REALTIME_BUS_BACKEND=postgres_notify
+# DASHBOARD_REALTIME_BUS_CHANNEL=lumonox_dashboard_realtime
+# Optional realtime snapshot + WS delta protocol (fallback remains POST /dashboard/query):
+# LUMONOX_DASHBOARD_REALTIME_ENABLED=false
+# LUMONOX_DASHBOARD_REALTIME_WS_ENABLED=false
+# LUMONOX_DASHBOARD_REALTIME_SNAPSHOT_MAX_PROJECTS=512
+# LUMONOX_DASHBOARD_REALTIME_SNAPSHOT_TTL_SECONDS=900
+# LUMONOX_DASHBOARD_REALTIME_MAX_DELTA_QUEUE_PER_PROJECT=32
+# LUMONOX_DASHBOARD_REALTIME_SNAPSHOT_RECONCILE_INTERVAL_SECONDS=15
+# LUMONOX_DASHBOARD_REALTIME_SNAPSHOT_MAX_DRIFT_VERSIONS=25
+# LUMONOX_DASHBOARD_QUERY_SNAPSHOT_REFRESH_SECONDS=30
+# LUMONOX_DASHBOARD_WS_LIVE_TICK_SECONDS=2
+# NEXT_PUBLIC_LUMONOX_LIVE_DELTA_REFRESH_THROTTLE_MS=800
+# NEXT_PUBLIC_LUMONOX_DASHBOARD_REALTIME_WS_ENABLED=false
+
+# Optional dashboard frontend RUM ingest guardrails (`POST /lumonox/rum`):
+# DASHBOARD_RUM_MAX_REQUEST_BYTES=8192
+# DASHBOARD_RUM_LOG_PAYLOADS=false
+
+# Browser origins allowed for credentialed dashboard calls (include every host:port you open the UI on).
+# Default in code also allows 8000/8000; override if you use other ports.
+# CORS_ALLOW_ORIGINS=http://127.0.0.1:8000,http://localhost:8000,http://127.0.0.1:8000,http://localhost:8000
+
+# Dashboard magic link — use the SAME email you type on the sign-in screen (or use DASHBOARD_ALLOWED_EMAIL_DOMAINS).
+# For local dev without email delivery, set DASHBOARD_AUTH_MAGIC_LINK_DEV_EXPOSE_TOKEN=true to get a link in the UI.
+DASHBOARD_AUTH_ALLOWED_EMAIL=you@example.com
+ALERT_EMAIL_PROVIDER=file
+# Relative paths are anchored to the project/data root (see resolve_lumonox_data_root), not cwd.
+ALERT_EMAIL_FILE_OUTBOX_DIR=./.lumonox/emails
+ALERT_EMAIL_FROM=alerts@example.com
+# DASHBOARD_AUTH_MAGIC_LINK_DEV_EXPOSE_TOKEN=true
+
+# Optional: static UI env bundle — often repo-root .env.lumonox for Next builds
+# NEXT_PUBLIC_LUMONOX_API_BASE_URL=/lumonox
+# NEXT_PUBLIC_LUMONOX_API_KEY=…
+
+# Optional: custom .env.lumonox path (default ./.env.lumonox)
+# LUMONOX_ENV_FILE=/path/to/.env.lumonox

lumonox-0.2.5/.gitignore

@@ -0,0 +1,73 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+/lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+.venv/
+venv/
+ENV/
+
+# uv
+.uv/
+
+# Testing / coverage
+.pytest_cache/
+.coverage
+htmlcov/
+.mypy_cache/
+.ruff_cache/
+
+# Node / Next.js (frontend)
+node_modules/
+.next/
+out/
+.turbo/
+*.tsbuildinfo
+
+# OS / editors
+.DS_Store
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Environment
+.env
+.env.*
+!.env.example
+!.env.lumonox.example
+
+# Local SQLite (dev)
+*.db
+
+# Logs
+*.log
+
+.lumonox/
+# Legacy local data dir name (pre-rename checkouts); never commit.
+.autopulse/
+backend/.autopulse/
+sdk/.autopulse/
+
+# Local stray copies of dashboard static export (canonical build is frontend/out)
+sdk/src/lumonox/ui/
+
+# Hatch sdist staging for shipped wheels (never commit baked export under src)
+backend/src/lumonox_backend/dashboard_static/

lumonox-0.2.5/ALERT_DELIVERY_RUNBOOK.md

@@ -0,0 +1,59 @@
+# Alert Delivery Verification Runbook
+
+This runbook validates that alert delivery is configured and observable from the dashboard.
+
+## 1) Configure sender mode
+
+Set one of the minimal sender configurations:
+
+- Email provider (recommended first):
+  - `ALERT_SENDER_MODE=email`
+  - `ALERT_EMAIL_PROVIDER=resend` (or `postmark`)
+  - `ALERT_EMAIL_API_KEY=...`
+  - `ALERT_EMAIL_FROM=alerts@example.com`
+- Email (zero-config dev outbox; writes `.eml` locally):
+  - `ALERT_SENDER_MODE=email`
+  - `ALERT_EMAIL_PROVIDER=file`
+  - optional: `ALERT_EMAIL_FILE_OUTBOX_DIR=./.lumonox/emails`
+  - optional: `ALERT_EMAIL_FROM=alerts@localhost`
+- Email (no SaaS, requires local MTA present):
+  - `ALERT_SENDER_MODE=email`
+  - `ALERT_EMAIL_PROVIDER=sendmail`
+  - optional: `ALERT_SENDMAIL_PATH=/usr/sbin/sendmail`
+- Email (no SaaS, requires SMTP server reachable):
+  - `ALERT_SENDER_MODE=email`
+  - `ALERT_EMAIL_PROVIDER=smtp`
+  - `ALERT_EMAIL_SMTP_HOST=127.0.0.1` (or your SMTP host)
+  - optional: `ALERT_EMAIL_SMTP_PORT=25`
+  - optional: `ALERT_EMAIL_SMTP_USE_TLS=true`
+  - optional: `ALERT_EMAIL_SMTP_USERNAME=...`
+  - optional: `ALERT_EMAIL_SMTP_PASSWORD=...`
+- Slack webhook:
+  - `ALERT_SENDER_MODE=slack`
+  - `ALERT_SLACK_WEBHOOK_URL=...`
+- Discord webhook:
+  - `ALERT_SENDER_MODE=discord`
+  - `ALERT_DISCORD_WEBHOOK_URL=...`
+- Multi-channel:
+  - `ALERT_SENDER_MODE=composite`
+  - combine email + Slack and/or Discord vars above.
+
+## 2) Trigger one evaluation pass
+
+```bash
+uv run python -m lumonox_backend.jobs alerts-once
+```
+
+The command prints the number of successfully sent alerts in that run.
+
+## 3) Validate dispatch observability
+
+Open the dashboard Alerts page and verify dispatch rows include:
+
+- `status` (`sent`, `failed`, or `skipped`)
+- `reason_code` for failures/skips
+- `attempt_count`
+- `delivered_at` (for successful sends)
+- `provider_message_id` when available
+
+Use the **Failed only** filter to quickly review actionable delivery failures.

lumonox-0.2.5/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: lumonox
+Version: 0.2.5
+Summary: Lumonox backend ingestion API
+Requires-Python: >=3.11
+Requires-Dist: aiosqlite>=0.22.1
+Requires-Dist: alembic>=1.14.0
+Requires-Dist: asyncpg>=0.30.0
+Requires-Dist: duckdb>=1.1.3
+Requires-Dist: fastapi>=0.115.0
+Requires-Dist: greenlet>=3.0.0
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: psutil>=6.0.0
+Requires-Dist: psycopg[binary]>=3.3.3
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: sqlalchemy>=2.0.36
+Requires-Dist: uvicorn>=0.32.0
+Requires-Dist: websockets>=12.0
+Provides-Extra: parquet-s3
+Requires-Dist: boto3>=1.34.0; extra == 'parquet-s3'

lumonox-0.2.5/README.md

@@ -0,0 +1,138 @@
+# Lumonox Backend
+
+FastAPI backend for ingest, dashboard APIs, auth/session flows, alerts, retention jobs, and realtime updates.
+
+Python **import name** is **`lumonox_backend`**. The **distribution / PyPI project name** is **`lumonox`** (API + pre-built dashboard static export bundled in the wheel). Use **`lumonox`** / **`lumonox-sdk`** for installs from PyPI.
+
+## What lives here
+
+- Ingest API: `POST /ingest` with API-key authentication.
+- Dashboard API: overview, requests, error groups, diagnosis, alerts, settings, log query, organizations.
+- Dashboard auth: magic-link sign-in, cookie sessions, tenant bootstrap endpoint.
+- Background jobs: alert evaluation and retention cleanup.
+- Internal ops endpoints: health/ready and service metrics.
+
+## Install outside the monorepo
+
+**One line (PyPI, after trusted publishing is enabled):**
+
+```bash
+pip install lumonox
+```
+
+```bash
+uv add lumonox
+```
+
+**One line (Git — always works; pin `main` to a tag or SHA in production):**
+
+```bash
+uv add "lumonox @ git+https://github.com/sintimaski/lumonox.git@main#subdirectory=backend"
+```
+
+```bash
+pip install "lumonox @ git+https://github.com/sintimaski/lumonox.git@main#subdirectory=backend"
+```
+
+**API + Fast instrumented app in one line:** use the SDK extra (see [sdk/README.md](../sdk/README.md)):
+
+```bash
+pip install "lumonox-sdk[stack]"
+```
+
+If Python **3.14** fails to resolve wheels, try **3.12 or 3.13**.
+
+### Wheel build (bundled dashboard)
+
+The **`lumonox`** sdist/wheel ships the Next static export under **`lumonox_backend/dashboard_static/`** (mounted at **`/lumonox/ui/`** when **`LUMONOX_FRONTEND_STATIC_DIR`** is unset and **`index.html`** exists). Build **`frontend/out`** first (see **`scripts/run_synthetic_stack.sh`** for **`NEXT_PUBLIC_*`** defaults):
+
+```bash
+./backend/scripts/package_wheel.sh
+```
+
+## Run locally
+
+From repository root:
+
+```bash
+uv sync --group dev
+# Loads backend/.env so retention/cap settings are actually applied.
+uv run uvicorn lumonox_backend.main:app --env-file backend/.env --log-level info
+```
+
+Backend defaults to `http://localhost:8000`.
+
+## Key environment variables
+
+- `DATABASE_URL` (default SQLite file: `.lumonox/lumonox.db` under the repo root—same directory tree as DuckDB; see `normalize_database_url` in `core/config.py`)
+- `LUMONOX_EVENT_STORE` (`duckdb` default; set `sqlite` to force legacy SQL event reads)
+- `LUMONOX_DUCKDB_PATH` (DuckDB event store file; relative values anchor under `LUMONOX_DATA_DIR` / `LUMONOX_PROJECT_ROOT`, else monorepo root—see `normalize_event_store_duckdb_path` / `resolve_lumonox_data_root` in `core/config.py`; use absolute paths in production if you prefer)
+- `LUMONOX_DATA_DIR` / `LUMONOX_PROJECT_ROOT` (optional; pins the root for relative DuckDB paths and keeps ingest/dashboard/CLI on one file regardless of shell cwd)
+- `CORS_ALLOW_ORIGINS`
+- `DASHBOARD_AUTH_ALLOWED_EMAIL`
+- `DASHBOARD_AUTH_ALLOW_API_KEY_FALLBACK` (disabled by default; enable only for controlled non-browser flows)
+- `INGEST_MAX_REQUEST_BYTES`
+- `INGEST_RATE_LIMIT_REQUESTS_PER_WINDOW`
+- `INGEST_RATE_LIMIT_WINDOW_SECONDS`
+- `INGEST_DISTRIBUTED_RATE_LIMIT_ENABLED` (enables DB-backed distributed limiter)
+- `INGEST_ASYNC_AGGREGATE_ENABLED` (keeps ingest hot path raw-write-first)
+- `INGEST_ASYNC_AGGREGATE_QUEUE_MAX_SIZE`
+- `INGEST_DROP_LUMONOX_TRAFFIC_FROM_DB` (drops `/lumonox/*`, `/dashboard/*`, and `/ingest` events before persistence)
+- `JOBS_ENABLE_SCHEDULER`
+- `JOBS_RETENTION_INTERVAL_SECONDS` (minimum **5**; periodic `run_retention_cleanup_once` when scheduler or retention-only loop runs)
+- `JOBS_SCHEDULER_LEASE_ENABLED` (prevents duplicate periodic job execution across instances)
+- `JOBS_SCHEDULER_LEASE_TTL_SECONDS`
+- `LUMONOX_ENV_FILE` (optional path to the `.env.lumonox` bundle for local static UI builds; default `./.env.lumonox` at process cwd)
+- `LUMONOX_SQLITE_MAX_DB_FILE_MB` (max SQLite log-store file size in MB; applies to DuckDB or SQLite when capped. For SQLite it includes main + `-wal` + `-shm`; deprecated alias `LUMONOX_EMBEDDED_MAX_DB_SIZE_MB`; default **512** on dev default SQLite filenames when unset)
+- `LUMONOX_RETENTION_PRESSURE_POLL_SECONDS` / `LUMONOX_RETENTION_PRESSURE_MIN_INTERVAL_SECONDS` (SQLite pressure poll; see `core/config.py`)
+
+Parquet **object storage** with `LUMONOX_PARQUET_OBJECT_STORAGE_URI=s3://...` needs **`boto3`**. Install the extra from this directory (`uv pip install -e ".[parquet-s3]"`) or add `boto3` to your environment. `file://` URIs do not use `boto3`.
+
+See `backend/src/lumonox_backend/core/config.py` and `backend/.env.example` for the complete list and defaults.
+
+**Production:** startup applies `validate_deployment_settings` for `LUMONOX_ENV=production`. Follow `docs/ops/PRODUCTION_DEPLOYMENT.md` for enforced HTTPS ingest, internal metrics token, CORS, dashboard session/magic-link TTL, OIDC/magic-link URL schemes, and related constraints. Automated checks: `backend/tests/test_deployment_settings.py`.
+
+## Testing (backend)
+
+- Unit-style deployment guardrails: `uv run pytest backend/tests/test_deployment_settings.py`
+- Integration tests under `backend/tests/` use `BACKEND_TEST_DATABASE_URL` when set; otherwise `uv run pytest` uses an **isolated session SQLite file** under pytest’s basetemp (see `backend/tests/conftest.py`). Override explicitly, e.g. `export BACKEND_TEST_DATABASE_URL=sqlite+aiosqlite:////tmp/lx-test.db`, when you want a fixed path or Postgres.
+- Ingest idempotency replay parity is required on Postgres (CI enforced). Local equivalent:
+  `export BACKEND_TEST_DATABASE_URL=postgresql+asyncpg://lumonox:lumonox@127.0.0.1:5432/lumonox_ci && uv run pytest backend/tests/test_ingest.py::test_ingest_idempotency_key_replays_accepted_without_duplicate_events -q`
+- Backend CI-equivalent one-command gate (same backend checks split across `python-sqlite` + `python-postgres` CI jobs):
+  `export BACKEND_TEST_DATABASE_URL=postgresql+asyncpg://lumonox:lumonox@127.0.0.1:5432/lumonox_ci && make check-python-ci`
+
+## Retention scheduling (FastAPI-optional)
+
+The portable unit of work is **one retention pass** (SQLite caps, time windows, aggregates trim):
+
+| How | Command / API |
+|-----|-----------------|
+| **CLI (any OS, no web server)** | `cd backend && uv run python -m lumonox_backend.jobs retention-once` |
+| **Sync Python (cron, Django command, systemd `ExecStart`)** | `from lumonox_backend.jobs import run_retention_sync` then `run_retention_sync()` after setting `DATABASE_URL` in the environment |
+| **In-process (FastAPI)** | Lifespan starts the scheduler or retention-only loop; optional SQLite pressure poll |
+
+For **Linux production** or **Django** (no FastAPI event loop), prefer **cron** or **systemd timers** calling the CLI or `run_retention_sync()` on the interval you want, and set `JOBS_ENABLE_SCHEDULER=false` on the API so you do not double-run retention in-process and from cron.
+
+## Event store migration helpers
+
+- Backfill SQL `events` rows into the DuckDB event store:
+
+```bash
+cd backend
+uv run python scripts/backfill_events_to_duckdb.py --batch-size 1000
+```
+
+Example cron every five minutes:
+
+```cron
+*/5 * * * * cd /path/to/lumonox/backend && /path/to/uv run python -m lumonox_backend.jobs retention-once >>/var/log/lumonox-retention.log 2>&1
+```
+
+## Operational runbooks
+
+- Alert delivery setup and verification: `backend/ALERT_DELIVERY_RUNBOOK.md`
+- Release/incident drill gates: `docs/runbooks/PHASE5_RELEASE_CHECKLIST.md` and `docs/runbooks/PHASE5_INCIDENT_DRILLS.md`
+
+## Scope and constraints
+
+Product/architecture source of truth remains `DEVELOPMENT.md`.

lumonox-0.2.5/alembic.ini

@@ -0,0 +1,39 @@
+[alembic]
+script_location = src/lumonox_backend/alembic
+path_separator = os
+prepend_sys_path = ./src
+# Overridden at runtime by alembic/env.py from get_settings() (loads backend/.env).
+# Default app DB is SQLite under .lumonox/; Postgres requires DATABASE_URL in the environment.
+sqlalchemy.url = sqlite:///./.lumonox/lumonox.db
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s

lumonox-0.2.5/pyproject.toml

@@ -0,0 +1,39 @@
+[project]
+name = "lumonox"
+version = "0.2.5"
+description = "Lumonox backend ingestion API"
+requires-python = ">=3.11"
+dependencies = [
+    "fastapi>=0.115.0",
+    "python-dotenv>=1.0.0",
+    "uvicorn>=0.32.0",
+    "websockets>=12.0",
+    "sqlalchemy>=2.0.36",
+    "greenlet>=3.0.0",
+    "asyncpg>=0.30.0",
+    "psycopg[binary]>=3.3.3",
+    "alembic>=1.14.0",
+    "aiosqlite>=0.22.1",
+    "duckdb>=1.1.3",
+    "psutil>=6.0.0",
+    "httpx>=0.27.0",
+]
+
+[project.optional-dependencies]
+# Parquet object storage with ``s3://`` URIs (``parquet_object_storage``); not needed for ``file://``.
+parquet-s3 = [
+    "boto3>=1.34.0",
+]
+
+[build-system]
+requires = ["hatchling>=1.26.0"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.sdist]
+# Ship the Next static export inside the extracted tree used for wheels (nested under ``packages``)—avoids editable-install
+# breakage from wheel-only forced paths while keeping ``frontend/out`` out of Git.
+[tool.hatch.build.targets.sdist.force-include]
+"../frontend/out" = "src/lumonox_backend/dashboard_static"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/lumonox_backend"]

lumonox-0.2.5/scripts/backfill_events_to_duckdb.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+import argparse
+import asyncio
+from datetime import UTC, datetime
+
+from sqlalchemy import select
+
+from lumonox_backend.database import AsyncSessionLocal
+from lumonox_backend.models import Event
+from lumonox_backend.services.duckdb_async import run_duckdb_write_sync
+from lumonox_backend.services.event_store import get_duckdb_event_store
+
+
+async def run_backfill(*, batch_size: int, since: datetime | None) -> int:
+    store = get_duckdb_event_store()
+    copied = 0
+    last_id = 0
+    while True:
+        async with AsyncSessionLocal() as session:
+            query = (
+                select(Event).where(Event.id > last_id).order_by(Event.id.asc()).limit(batch_size)
+            )
+            if since is not None:
+                query = query.where(Event.received_at >= since)
+            rows = (await session.scalars(query)).all()
+        if not rows:
+            break
+        payload = [
+            {
+                "project_id": row.project_id,
+                "timestamp": row.timestamp,
+                "received_at": row.received_at,
+                "sdk_version": row.sdk_version,
+                "type": row.type,
+                "service_name": row.service_name,
+                "environment": row.environment,
+                "method": row.method,
+                "path": row.path,
+                "status_code": row.status_code,
+                "latency_ms": row.latency_ms,
+                "payload": row.payload,
+                "request_id": row.request_id,
+            }
+            for row in rows
+        ]
+        await run_duckdb_write_sync(store.insert_rows, payload)
+        copied += len(rows)
+        last_id = int(rows[-1].id)
+    return copied
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Backfill SQL events into DuckDB event store")
+    parser.add_argument("--batch-size", type=int, default=1000)
+    parser.add_argument(
+        "--since",
+        type=str,
+        default="",
+        help="Optional ISO timestamp, e.g. 2026-04-01T00:00:00Z",
+    )
+    return parser.parse_args()
+
+
+def _parse_since(raw: str) -> datetime | None:
+    value = raw.strip()
+    if not value:
+        return None
+    parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    if parsed.tzinfo is None:
+        return parsed.replace(tzinfo=UTC)
+    return parsed.astimezone(UTC)
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    total = asyncio.run(
+        run_backfill(
+            batch_size=max(1, int(args.batch_size)),
+            since=_parse_since(args.since),
+        )
+    )
+    print(f"backfilled_events={total}")

lumonox-0.2.5/scripts/debug_retention_vacuum.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import argparse
+import asyncio
+import sqlite3
+import time
+from pathlib import Path
+
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from lumonox_backend.core.config import get_settings
+from lumonox_backend.database import get_engine
+from lumonox_backend.jobs import run_retention_once
+from lumonox_backend.maintenance.retention import (
+    _resolve_sqlite_db_path,
+    _sqlite_db_disk_footprint_bytes,
+    sqlite_retention_pressure_pending,
+)
+from lumonox_backend.models import Event
+
+
+def _fmt_mb(value: int) -> str:
+    return f"{value / (1024 * 1024):.2f}MB"
+
+
+def _sqlite_vacuum_probe(db_path: Path) -> tuple[bool, str]:
+    try:
+        with sqlite3.connect(str(db_path), timeout=2.0) as connection:
+            connection.execute("PRAGMA busy_timeout=2000")
+            connection.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+            connection.execute("VACUUM")
+        return True, "ok"
+    except sqlite3.OperationalError as exc:
+        return False, str(exc)
+
+
+async def _run(iterations: int, sleep_seconds: float, dispose_engine_after: bool) -> None:
+    settings = get_settings()
+    db_path = _resolve_sqlite_db_path(settings.database_url)
+    if db_path is None:
+        raise SystemExit(f"DATABASE_URL is not file-backed sqlite: {settings.database_url}")
+
+    engine = get_engine(settings.database_url)
+    session_maker = async_sessionmaker(bind=engine, expire_on_commit=False, class_=AsyncSession)
+    print(f"db={db_path}")
+    print(
+        f"cap_mb={settings.sqlite_max_db_file_mb} "
+        f"interval={settings.jobs_retention_interval_seconds} "
+        f"pressure_poll={settings.retention_pressure_poll_seconds}"
+    )
+
+    for i in range(iterations):
+        async with session_maker() as session:
+            pending = await sqlite_retention_pressure_pending(session, settings)
+            event_count = int((await session.execute(select(func.count(Event.id)))).scalar_one())
+        before = _sqlite_db_disk_footprint_bytes(db_path)
+        start = time.monotonic()
+        deleted = await run_retention_once(
+            settings=settings, dispose_engine_after=dispose_engine_after
+        )
+        elapsed_ms = int((time.monotonic() - start) * 1000)
+        after = _sqlite_db_disk_footprint_bytes(db_path)
+        probe_ok, probe_msg = _sqlite_vacuum_probe(db_path)
+        print(
+            f"[{i + 1}/{iterations}] pending={pending} events={event_count} deleted={deleted} "
+            f"before={_fmt_mb(before)} after={_fmt_mb(after)} elapsed_ms={elapsed_ms} "
+            f"vacuum_probe={'ok' if probe_ok else 'locked'} ({probe_msg})"
+        )
+        if i < iterations - 1:
+            await asyncio.sleep(sleep_seconds)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Debug scheduled retention + VACUUM behavior.")
+    parser.add_argument(
+        "--iterations", type=int, default=8, help="How many retention passes to run."
+    )
+    parser.add_argument(
+        "--sleep",
+        type=float,
+        default=2.0,
+        help="Sleep seconds between passes.",
+    )
+    parser.add_argument(
+        "--dispose-engine-after",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+        help="Dispose shared async engine before VACUUM (default: true).",
+    )
+    args = parser.parse_args()
+    asyncio.run(_run(args.iterations, args.sleep, args.dispose_engine_after))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())