nilalytics 0.1.0__py3-none-any.whl

nilalytics/__init__.py

@@ -0,0 +1 @@
+"""nilalytics: serverless realtime analytics on DuckLake + Quack + object storage."""

nilalytics/autocapture.py

@@ -0,0 +1,114 @@
+"""Automatic error submission for Python apps/services.
+
+Call ``install()`` once at startup. After that, uncaught exceptions (main
+thread and worker threads) and ``logger.exception()`` / ``logging.error(...,
+exc_info=True)`` calls are sent to nilalytics as OTLP error logs
+(severity ERROR + ``exception.*`` attributes) — the same shape Grafana Faro
+and Sentry use. No per-error code needed.
+
+Usage:
+    from nilalytics import autocapture
+    autocapture.install("my-service")
+    ...                       # any uncaught error now self-reports
+
+You can also report a handled error explicitly:
+    try:
+        risky()
+    except Exception as exc:
+        autocapture.report(exc, order_id=123)
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+import threading
+import time
+import traceback
+
+import requests
+
+from . import config
+
+_service = "nilalytics-app"
+
+
+def _attr(key: str, value: str) -> dict:
+    return {"key": key, "value": {"stringValue": str(value)}}
+
+
+def _error_record(exc_type, exc_value, exc_tb, extra: dict | None = None) -> dict:
+    stack = "".join(traceback.format_exception(exc_type, exc_value, exc_tb))
+    attrs = [
+        _attr("event.name", "exception"),
+        _attr("exception.type", getattr(exc_type, "__name__", str(exc_type))),
+        _attr("exception.message", str(exc_value)),
+        _attr("exception.stacktrace", stack),
+    ]
+    attrs += [_attr(k, v) for k, v in (extra or {}).items()]
+    return {
+        "timeUnixNano": str(time.time_ns()),
+        "severityNumber": 17,  # ERROR
+        "severityText": "ERROR",
+        "body": {"stringValue": "exception"},
+        "attributes": attrs,
+    }
+
+
+def _post(record: dict) -> None:
+    payload = {
+        "resourceLogs": [
+            {
+                "resource": {"attributes": [_attr("service.name", _service)]},
+                "scopeLogs": [{"scope": {"name": "nilalytics.autocapture"},
+                               "logRecords": [record]}],
+            }
+        ]
+    }
+    try:
+        requests.post(
+            f"{config.OTLP_HTTP}/v1/logs",
+            json=payload,
+            headers={"Authorization": f"Bearer {config.OTLP_TOKEN}"},
+            timeout=5,
+        )
+    except Exception:
+        # Telemetry must never crash the app it is observing.
+        pass
+
+
+def report(exc: BaseException, **extra) -> None:
+    """Submit a handled exception explicitly."""
+    _post(_error_record(type(exc), exc, exc.__traceback__, extra))
+
+
+class _LoggingHandler(logging.Handler):
+    def emit(self, record: logging.LogRecord) -> None:
+        if record.exc_info:
+            exc_type, exc_value, exc_tb = record.exc_info
+            _post(_error_record(exc_type, exc_value, exc_tb,
+                                {"logger": record.name, "log.message": record.getMessage()}))
+
+
+def install(service_name: str = "nilalytics-app") -> None:
+    """Install global handlers so all uncaught errors self-report."""
+    global _service
+    _service = service_name
+
+    # 1) uncaught exceptions on the main thread
+    _prev_hook = sys.excepthook
+
+    def _excepthook(exc_type, exc_value, exc_tb):
+        _post(_error_record(exc_type, exc_value, exc_tb))
+        _prev_hook(exc_type, exc_value, exc_tb)
+
+    sys.excepthook = _excepthook
+
+    # 2) uncaught exceptions on worker threads
+    def _threadhook(args):
+        _post(_error_record(args.exc_type, args.exc_value, args.exc_traceback))
+
+    threading.excepthook = _threadhook
+
+    # 3) logger.exception() / logging.error(..., exc_info=True)
+    logging.getLogger().addHandler(_LoggingHandler())

nilalytics/cli.py

@@ -0,0 +1,60 @@
+"""Unified nilalytics command-line interface.
+
+After installing the package (``pip install nilalytics`` / ``uv sync``), the
+``nilalytics`` command is available anywhere:
+
+    nilalytics server                 # ingest + Quack catalog server
+    nilalytics gateway                # public ingest gateway (CORS, tokens, TLS)
+    nilalytics emit --persons 5       # send sample events
+    nilalytics query report           # analytics report over Quack
+    nilalytics maintenance --expire   # flush inlined data + compact
+
+Each subcommand delegates to its module, which is also runnable directly via
+``python -m nilalytics.<module>``.
+"""
+
+from __future__ import annotations
+
+import sys
+
+_USAGE = """usage: nilalytics <command> [args]
+
+commands:
+  server                      run the ingest + Quack catalog server
+  gateway                     run the public ingest gateway (CORS, short-lived tokens, TLS)
+  emit [options]              send sample logs/traces/metrics (--persons for cross-device)
+  query [subcommand] [args]   report | traces | metrics | stitch | asof | changes | snapshots | errors
+  maintenance [--expire]      flush inlined data to Parquet + compact
+"""
+
+
+def main(argv: list[str] | None = None) -> int:
+    argv = list(sys.argv[1:] if argv is None else argv)
+    if not argv or argv[0] in ("-h", "--help", "help"):
+        print(_USAGE)
+        return 0
+
+    command, rest = argv[0], argv[1:]
+    if command == "server":
+        from . import server
+        server.main(rest)
+    elif command == "gateway":
+        from . import gateway
+        gateway.main(rest)
+    elif command == "emit":
+        from . import emitter
+        emitter.main(rest)
+    elif command == "query":
+        from . import query
+        query.main(rest)
+    elif command == "maintenance":
+        from . import maintenance
+        maintenance.main(rest)
+    else:
+        print(f"unknown command: {command}\n\n{_USAGE}")
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

nilalytics/config.py

@@ -0,0 +1,160 @@
+"""Central configuration for nilalytics.
+
+Every value can be overridden with an environment variable so the same code runs
+against local MinIO now and Cloudflare R2 later (R2 is S3-compatible).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import secrets
+import stat
+from pathlib import Path
+
+# Writable data directory (catalog file + secrets). Deliberately NOT relative to
+# the package install location, so nilalytics works from any working directory
+# and when pip-installed. Override with NILA_DATA_DIR.
+DATA_DIR = Path(os.getenv("NILA_DATA_DIR", Path.home() / ".nilalytics")).expanduser()
+DATA_DIR.mkdir(parents=True, exist_ok=True)
+
+# --- Secrets: never hardcode. Load from env, else generate once and persist
+# to a 0600 file under data/ (which is git-ignored). token_urlsafe() is safe in
+# both HTTP headers and single-quoted SQL strings. ---
+_SECRETS_FILE = DATA_DIR / ".nila_secrets.json"
+
+
+def _load_or_create_secrets() -> dict:
+    data = {}
+    if _SECRETS_FILE.exists():
+        try:
+            data = json.loads(_SECRETS_FILE.read_text())
+        except (ValueError, OSError):
+            data = {}
+    changed = False
+    for key in ("otlp_token", "quack_token", "id_salt", "gateway_secret", "ingest_key"):
+        if not data.get(key):
+            data[key] = secrets.token_urlsafe(24)
+            changed = True
+    if changed:
+        _SECRETS_FILE.write_text(json.dumps(data))
+        try:
+            _SECRETS_FILE.chmod(stat.S_IRUSR | stat.S_IWUSR)  # 0600, owner-only
+        except OSError:
+            pass
+    return data
+
+
+_SECRETS = _load_or_create_secrets()
+
+# --- Object storage backend (works on all major clouds) ---
+# NILA_STORAGE selects the cloud: s3 | gcs | r2 | azure. See storage.py.
+STORAGE = os.getenv("NILA_STORAGE", "s3").lower()
+# Bucket (or Azure *container*) + key prefix, shared across backends.
+BUCKET = os.getenv("NILA_BUCKET", os.getenv("NILA_S3_BUCKET", "nilalytics"))
+PREFIX = os.getenv("NILA_PREFIX", "lake")
+
+# S3 / MinIO / any S3-compatible store. Leave endpoint empty for real AWS S3.
+S3_ENDPOINT = os.getenv("NILA_S3_ENDPOINT", "127.0.0.1:9100")
+S3_ACCESS_KEY = os.getenv("NILA_S3_KEY", "minioadmin")
+S3_SECRET_KEY = os.getenv("NILA_S3_SECRET", "minioadmin")
+S3_SESSION_TOKEN = os.getenv("NILA_S3_SESSION_TOKEN", "")  # optional STS token
+S3_USE_SSL = os.getenv("NILA_S3_USE_SSL", "false").lower() == "true"
+S3_URL_STYLE = os.getenv("NILA_S3_URL_STYLE", "path")  # MinIO/R2 want path style
+S3_REGION = os.getenv("NILA_S3_REGION", "us-east-1")
+
+# Google Cloud Storage (HMAC interoperability keys).
+GCS_KEY = os.getenv("NILA_GCS_KEY", "")
+GCS_SECRET = os.getenv("NILA_GCS_SECRET", "")
+
+# Cloudflare R2.
+R2_ACCOUNT_ID = os.getenv("NILA_R2_ACCOUNT_ID", "")
+R2_KEY = os.getenv("NILA_R2_KEY", "")
+R2_SECRET = os.getenv("NILA_R2_SECRET", "")
+
+# Azure Blob / ADLS Gen2 (BUCKET is the container).
+AZURE_ACCOUNT = os.getenv("NILA_AZURE_ACCOUNT", "")
+AZURE_AUTH = os.getenv("NILA_AZURE_AUTH", "credential_chain")  # credential_chain|connection_string|service_principal
+AZURE_CONNECTION_STRING = os.getenv("NILA_AZURE_CONNECTION_STRING", "")
+AZURE_TENANT_ID = os.getenv("NILA_AZURE_TENANT_ID", "")
+AZURE_CLIENT_ID = os.getenv("NILA_AZURE_CLIENT_ID", "")
+AZURE_CLIENT_SECRET = os.getenv("NILA_AZURE_CLIENT_SECRET", "")
+
+# --- DuckLake catalog (a DuckDB file, served to clients over Quack) ---
+CATALOG_PATH = os.getenv("NILA_CATALOG", str(DATA_DIR / "catalog.ducklake"))
+# Inserts smaller than this land as rows inside the catalog (data inlining),
+# so streaming events never create tiny Parquet files.
+DATA_INLINING_ROW_LIMIT = int(os.getenv("NILA_INLINE_LIMIT", "1000"))
+
+# --- OTLP ingest server (duckdb-otlp) ---
+OTLP_URI = os.getenv("NILA_OTLP_URI", "otlp:127.0.0.1:4318")
+OTLP_HTTP = os.getenv("NILA_OTLP_HTTP", "http://127.0.0.1:4318")
+# Bearer token that ingest clients (emitter, autocapture, browser) must present.
+# This is the INTERNAL token; it never leaves the host once the gateway is used.
+OTLP_TOKEN = os.getenv("NILA_OTLP_TOKEN", _SECRETS["otlp_token"])
+
+# --- Public ingest gateway (front door for browsers + mobile) ---
+# Set host to 0.0.0.0 to accept real devices/browsers. Optional TLS via cert/key.
+GATEWAY_HOST = os.getenv("NILA_GATEWAY_HOST", "127.0.0.1")
+GATEWAY_PORT = int(os.getenv("NILA_GATEWAY_PORT", "4319"))
+GATEWAY_CORS_ORIGINS = os.getenv("NILA_GATEWAY_CORS", "*")
+GATEWAY_CERT = os.getenv("NILA_GATEWAY_CERT", "")
+GATEWAY_KEY = os.getenv("NILA_GATEWAY_KEY", "")
+# Short-lived client tokens are HMAC-signed with this secret and expire after TTL.
+GATEWAY_SECRET = os.getenv("NILA_GATEWAY_SECRET", _SECRETS["gateway_secret"])
+GATEWAY_TOKEN_TTL = int(os.getenv("NILA_GATEWAY_TOKEN_TTL", "900"))  # 15 min
+# The (rotatable) key an app presents to MINT a short-lived token. This is the
+# only value that ships in a client; it grants minting, not ingest or reads.
+GATEWAY_INGEST_KEY = os.getenv("NILA_INGEST_KEY", _SECRETS["ingest_key"])
+
+# --- Quack catalog server (read path for clients / DuckDB-WASM) ---
+QUACK_URI = os.getenv("NILA_QUACK_URI", "quack:localhost")
+QUACK_TOKEN = os.getenv("NILA_QUACK_TOKEN", _SECRETS["quack_token"])
+
+# Name the DuckLake is attached as, inside the server process.
+LAKE = "lake"
+
+# --- Ingest tuning (package defaults; override via env) ---
+# Freshness: force a commit when the oldest buffered row hits this age.
+SEAL_MAX_AGE_MS = int(os.getenv("NILA_SEAL_MAX_AGE_MS", "1000"))
+# Retention: how long snapshots/old files are kept before maintenance reclaims them.
+MAINTENANCE_RETENTION_MS = int(os.getenv("NILA_RETENTION_MS", str(7 * 24 * 3600 * 1000)))
+# Resource attributes lifted into first-class columns at ingest for pruning
+# (comma-separated; empty disables). Event-level fields stay in log_attributes.
+PROMOTE_RESOURCE_ATTRS = os.getenv("NILA_PROMOTE_RESOURCE_ATTRS", "deployment.environment")
+
+# Salt for pseudonymous identity hashing. User keys (email/account id) are hashed
+# with this salt client-side, so the lake stores a stable person-key it cannot
+# reverse into a real identity.
+ID_SALT = os.getenv("NILA_ID_SALT", _SECRETS["id_salt"])
+
+# --- Physical layout applied to the events table (query performance) ---
+# Partition transforms: identity | bucket(N,col) | year|month|day|hour(ts).
+PARTITION_BY = os.getenv("NILA_PARTITION_BY", "day(time_unix_nano)")
+SORTED_BY = os.getenv("NILA_SORTED_BY", "body, time_unix_nano")
+
+# --- Safety guardrails ---
+_LOCAL_HOSTS = {"127.0.0.1", "localhost", "::1", "[::1]"}
+
+
+def endpoint_host() -> str:
+    return S3_ENDPOINT.split(":")[0]
+
+
+def is_local_endpoint() -> bool:
+    return endpoint_host() in _LOCAL_HOSTS
+
+
+def assert_safe() -> None:
+    """Refuse obviously-insecure configurations before starting the server."""
+    # Only the S3 backend has the local-MinIO / default-cred footgun. GCS, R2 and
+    # Azure are inherently remote + TLS and are validated in storage.py.
+    if STORAGE != "s3" or is_local_endpoint():
+        return
+    problems = []
+    if not S3_USE_SSL:
+        problems.append("remote S3 requires TLS (set NILA_S3_USE_SSL=true)")
+    if S3_ACCESS_KEY == "minioadmin" or S3_SECRET_KEY == "minioadmin":
+        problems.append("default 'minioadmin' credentials must not be used with a remote endpoint")
+    if problems:
+        raise SystemExit("[nilalytics] refusing to start:\n  - " + "\n  - ".join(problems))

nilalytics/emitter.py

@@ -0,0 +1,233 @@
+"""nilalytics emitter: sends OTLP/HTTP log records to the ingest endpoint.
+
+This mimics what a Grafana Faro / OpenTelemetry browser SDK emits: product
+events and errors are both OTLP *log records*. Product events carry an
+``event.name`` attribute; errors use severity ``ERROR`` plus ``exception.*``
+attributes. The server (duckdb-otlp) lands them in the DuckLake ``otlp_logs``
+table.
+
+Run it:  uv run python -m nilalytics.emitter --count 200
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import os
+import random
+import time
+import uuid
+
+import requests
+
+from . import config
+
+PRODUCT_EVENTS = ["page_view", "signup_start", "signup_complete", "add_to_cart", "purchase"]
+PAGES = ["/", "/pricing", "/product", "/checkout", "/docs"]
+ENV = os.getenv("NILA_ENV", "production")
+HEADERS = {"Authorization": f"Bearer {config.OTLP_TOKEN}"}
+
+# A pool of "devices" (each an anonymous.id) that normal traffic comes from.
+DEVICE_POOL = [uuid.uuid4().hex for _ in range(30)]
+_SESSIONS: dict[str, str] = {}
+
+
+def new_anonymous_id() -> str:
+    return uuid.uuid4().hex
+
+
+def session_for(device: str) -> str:
+    # Occasionally roll a new session for the same device.
+    if device not in _SESSIONS or random.random() < 0.1:
+        _SESSIONS[device] = uuid.uuid4().hex[:16]
+    return _SESSIONS[device]
+
+
+def hash_key(raw_key: str) -> str:
+    """Pseudonymous person-key: a salted hash, done client-side. Never send raw."""
+    return hashlib.sha256((config.ID_SALT + raw_key).encode()).hexdigest()[:32]
+
+
+def _resource() -> dict:
+    # deployment.environment is promoted to a first-class column at ingest.
+    return {"attributes": [_attr("service.name", "nilalytics-web"),
+                           _attr("deployment.environment", ENV)]}
+
+
+def _attr(key: str, value: str) -> dict:
+    return {"key": key, "value": {"stringValue": str(value)}}
+
+
+def _record(event_name: str, anon: str, session: str, severity_text: str,
+            severity_number: int, user: str | None = None, extra: dict | None = None) -> dict:
+    # anonymous.id identifies a device; user.id (hashed) is set only once known.
+    attrs = [_attr("event.name", event_name), _attr("anonymous.id", anon), _attr("session.id", session)]
+    if user:
+        attrs.append(_attr("user.id", user))
+    attrs += [_attr(k, v) for k, v in (extra or {}).items()]
+    return {
+        "timeUnixNano": str(time.time_ns()),
+        "severityNumber": severity_number,
+        "severityText": severity_text,
+        "body": {"stringValue": event_name},
+        "attributes": attrs,
+    }
+
+
+def _payload(records: list[dict]) -> dict:
+    return {
+        "resourceLogs": [
+            {
+                "resource": _resource(),
+                "scopeLogs": [
+                    {
+                        "scope": {"name": "nilalytics.browser"},
+                        "logRecords": records,
+                    }
+                ],
+            }
+        ]
+    }
+
+
+def make_events(n: int, error_rate: float) -> list[dict]:
+    records = []
+    for _ in range(n):
+        device = random.choice(DEVICE_POOL)
+        session = session_for(device)
+        if random.random() < error_rate:
+            records.append(
+                _record(
+                    "exception", device, session, "ERROR", 17,
+                    extra={
+                        "exception.type": random.choice(["TypeError", "NetworkError", "RangeError"]),
+                        "exception.message": "Cannot read properties of undefined",
+                        "page": random.choice(PAGES),
+                    },
+                )
+            )
+        else:
+            records.append(
+                _record(
+                    random.choice(PRODUCT_EVENTS), device, session, "INFO", 9,
+                    extra={"page": random.choice(PAGES)},
+                )
+            )
+    return records
+
+
+def emit(count: int, batch_size: int, error_rate: float) -> int:
+    url = f"{config.OTLP_HTTP}/v1/logs"
+    sent = 0
+    while sent < count:
+        this = min(batch_size, count - sent)
+        resp = requests.post(url, json=_payload(make_events(this, error_rate)),
+                             headers=HEADERS, timeout=10)
+        resp.raise_for_status()
+        sent += this
+        print(f"POST {this:>4} logs -> {resp.status_code} {resp.json()}", flush=True)
+    print(f"[emitter] sent {sent} log events to {url}", flush=True)
+    return sent
+
+
+def _post_logs(records: list[dict]) -> None:
+    resp = requests.post(f"{config.OTLP_HTTP}/v1/logs", json=_payload(records), headers=HEADERS, timeout=10)
+    resp.raise_for_status()
+
+
+def identify(device: str, user_key: str) -> str:
+    """Link a device's anonymous.id to a hashed person-key (an 'identify' event)."""
+    uid = hash_key(user_key)
+    _post_logs([_record("identify", device, session_for(device), "INFO", 9, user=uid, extra={"method": "login"})])
+    return uid
+
+
+def emit_cross_device(persons: int, events_per_device: int = 8) -> None:
+    """Simulate each person using two devices, then logging in on both.
+
+    Before login the two devices are unlinkable; the identify events tie both
+    anonymous.ids to the same hashed person-key, enabling cross-device stitching.
+    """
+    for i in range(persons):
+        user_key = f"person{i}@example.com"
+        uid = hash_key(user_key)
+        for _ in range(2):  # phone + laptop
+            device = new_anonymous_id()
+            session = uuid.uuid4().hex[:16]
+            # anonymous activity first
+            _post_logs([_record(random.choice(PRODUCT_EVENTS), device, session, "INFO", 9,
+                                extra={"page": random.choice(PAGES)}) for _ in range(events_per_device)])
+            # then the user logs in on this device
+            identify(device, user_key)
+        print(f"[emitter] person{i}: 2 devices -> {uid[:12]}...", flush=True)
+    print(f"[emitter] cross-device sim: {persons} persons x 2 devices", flush=True)
+
+
+# --- Traces (performance spans, e.g. page loads / API calls) ---
+def make_spans(n: int) -> list[dict]:
+    spans = []
+    for _ in range(n):
+        start = time.time_ns()
+        duration_ns = random.randint(20, 800) * 1_000_000  # 20-800 ms
+        spans.append({
+            "traceId": os.urandom(16).hex(),
+            "spanId": os.urandom(8).hex(),
+            "name": random.choice(["page_load", "api_call", "route_change"]),
+            "kind": 1,
+            "startTimeUnixNano": str(start),
+            "endTimeUnixNano": str(start + duration_ns),
+            "attributes": [_attr("page", random.choice(PAGES))],
+        })
+    return spans
+
+
+def emit_traces(n: int) -> None:
+    payload = {"resourceSpans": [{"resource": _resource(),
+                                  "scopeSpans": [{"scope": {"name": "nilalytics.browser"},
+                                                  "spans": make_spans(n)}]}]}
+    resp = requests.post(f"{config.OTLP_HTTP}/v1/traces", json=payload, headers=HEADERS, timeout=10)
+    resp.raise_for_status()
+    print(f"[emitter] sent {n} spans -> {resp.status_code}", flush=True)
+
+
+# --- Metrics (e.g. web-vitals gauges) ---
+def make_points(n: int) -> list[dict]:
+    return [{"timeUnixNano": str(time.time_ns()),
+             "asDouble": round(random.uniform(200, 4000), 1),
+             "attributes": [_attr("page", random.choice(PAGES))]} for _ in range(n)]
+
+
+def emit_metrics(n: int) -> None:
+    payload = {"resourceMetrics": [{"resource": _resource(),
+                                    "scopeMetrics": [{"scope": {"name": "nilalytics.browser"},
+                                                      "metrics": [{"name": "web_vitals_lcp_ms",
+                                                                   "unit": "ms",
+                                                                   "gauge": {"dataPoints": make_points(n)}}]}]}]}
+    resp = requests.post(f"{config.OTLP_HTTP}/v1/metrics", json=payload, headers=HEADERS, timeout=10)
+    resp.raise_for_status()
+    print(f"[emitter] sent {n} metric points -> {resp.status_code}", flush=True)
+
+
+def main(argv=None) -> None:
+    parser = argparse.ArgumentParser(prog="nilalytics emit",
+                                     description="Emit OTLP logs, traces and metrics to nilalytics.")
+    parser.add_argument("-n", "--count", type=int, default=200, help="log events")
+    parser.add_argument("-b", "--batch-size", type=int, default=50)
+    parser.add_argument("-e", "--error-rate", type=float, default=0.1)
+    parser.add_argument("--traces", type=int, default=10, help="trace spans")
+    parser.add_argument("--metrics", type=int, default=10, help="metric points")
+    parser.add_argument("--persons", type=int, default=0,
+                        help="simulate N cross-device persons (2 devices each, identified)")
+    args = parser.parse_args(argv)
+    if args.count:
+        emit(args.count, args.batch_size, args.error_rate)
+    if args.traces:
+        emit_traces(args.traces)
+    if args.metrics:
+        emit_metrics(args.metrics)
+    if args.persons:
+        emit_cross_device(args.persons)
+
+
+if __name__ == "__main__":
+    main()