devlogs 2.2.8__tar.gz → 2.3.0__tar.gz

{devlogs-2.2.8/src/devlogs.egg-info → devlogs-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devlogs
-Version: 2.2.8
+Version: 2.3.0
 Summary: Developer-focused logging library for Python with OpenSearch integration.
 Author-email: Dan Driscoll <dan@thedandriscoll.org>
 License: MIT License
@@ -62,6 +62,10 @@ If you don't have OpenSearch running and you want to stand one up:
 
 ## Step 2: Copy/paste these instructions into your coding agent
 
+Pick the block for your language and paste it into your coding agent. The agent will install devlogs as a dev dependency, create a connection config file, initialize the index, and add a small logging hook to your app entrypoint — guarded by an environment check so it only runs in development. No existing code is modified.
+
+For language-specific details and quirks, see [AGENT_HOWTO_PYTHON.md](AGENT_HOWTO_PYTHON.md) or [AGENT_HOWTO_JAVASCRIPT.md](AGENT_HOWTO_JAVASCRIPT.md).
+
 ### Python
 
 > Please do the following in this project:
@@ -109,9 +113,9 @@ If you don't have OpenSearch running and you want to stand one up:
 >        application: 'my-app',   // Required: your app name
 >        component: 'frontend',   // Required: component name
 >      });
+>      devlogs.installGlobalHandlers();
 >    }
 >    ```
->    After `init()`, all `console.log`, `console.warn`, `console.error`, and `console.debug` calls are automatically forwarded to OpenSearch. The original console output is preserved.
 > 3. Use `devlogs.setArea('dashboard')` and `devlogs.setOperationId('op-123')` to add context to logs. Pass a plain object as the last argument to attach custom fields:
 >    ```javascript
 >    console.log('User action', { userId: 123, action: 'clicked' });
@@ -459,6 +463,8 @@ See [docs/build-info.md](docs/build-info.md) for CI integration examples and ful
 
 ## See Also
 
+- [AGENT_HOWTO_PYTHON.md](AGENT_HOWTO_PYTHON.md) - Agent setup instructions for Python
+- [AGENT_HOWTO_JAVASCRIPT.md](AGENT_HOWTO_JAVASCRIPT.md) - Agent setup instructions for browser JS/TS
 - [MIGRATION-V2.md](MIGRATION-V2.md) - Migration guide from v1.x to v2.0
 - [HOWTO-COLLECTOR.md](HOWTO-COLLECTOR.md) - HTTP collector setup and deployment
 - [HOWTO-DEVLOGS-FORMAT.md](HOWTO-DEVLOGS-FORMAT.md) - Devlogs record format reference

{devlogs-2.2.8 → devlogs-2.3.0}/README.md

@@ -19,6 +19,10 @@ If you don't have OpenSearch running and you want to stand one up:
 
 ## Step 2: Copy/paste these instructions into your coding agent
 
+Pick the block for your language and paste it into your coding agent. The agent will install devlogs as a dev dependency, create a connection config file, initialize the index, and add a small logging hook to your app entrypoint — guarded by an environment check so it only runs in development. No existing code is modified.
+
+For language-specific details and quirks, see [AGENT_HOWTO_PYTHON.md](AGENT_HOWTO_PYTHON.md) or [AGENT_HOWTO_JAVASCRIPT.md](AGENT_HOWTO_JAVASCRIPT.md).
+
 ### Python
 
 > Please do the following in this project:
@@ -66,9 +70,9 @@ If you don't have OpenSearch running and you want to stand one up:
 >        application: 'my-app',   // Required: your app name
 >        component: 'frontend',   // Required: component name
 >      });
+>      devlogs.installGlobalHandlers();
 >    }
 >    ```
->    After `init()`, all `console.log`, `console.warn`, `console.error`, and `console.debug` calls are automatically forwarded to OpenSearch. The original console output is preserved.
 > 3. Use `devlogs.setArea('dashboard')` and `devlogs.setOperationId('op-123')` to add context to logs. Pass a plain object as the last argument to attach custom fields:
 >    ```javascript
 >    console.log('User action', { userId: 123, action: 'clicked' });
@@ -416,6 +420,8 @@ See [docs/build-info.md](docs/build-info.md) for CI integration examples and ful
 
 ## See Also
 
+- [AGENT_HOWTO_PYTHON.md](AGENT_HOWTO_PYTHON.md) - Agent setup instructions for Python
+- [AGENT_HOWTO_JAVASCRIPT.md](AGENT_HOWTO_JAVASCRIPT.md) - Agent setup instructions for browser JS/TS
 - [MIGRATION-V2.md](MIGRATION-V2.md) - Migration guide from v1.x to v2.0
 - [HOWTO-COLLECTOR.md](HOWTO-COLLECTOR.md) - HTTP collector setup and deployment
 - [HOWTO-DEVLOGS-FORMAT.md](HOWTO-DEVLOGS-FORMAT.md) - Devlogs record format reference

{devlogs-2.2.8 → devlogs-2.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "devlogs"
-version = "2.2.8"
+version = "2.3.0"
 description = "Developer-focused logging library for Python with OpenSearch integration."
 requires-python = ">=3.11"
 readme = "README.md"

{devlogs-2.2.8 → devlogs-2.3.0}/src/devlogs/_version_static.py

@@ -1,2 +1,2 @@
 # AUTO-GENERATED at build time — do not edit or commit
-__version__ = "2.2.8"
+__version__ = "2.3.0"

{devlogs-2.2.8 → devlogs-2.3.0}/src/devlogs/cli.py

@@ -28,7 +28,7 @@ from .opensearch.mappings import (
 	build_reindex_script,
 	SCHEMA_VERSION,
 )
-from .opensearch.queries import normalize_log_entries, search_logs, tail_logs, get_last_errors
+from .opensearch.queries import normalize_log_entries, search_logs, tail_logs, get_last_errors, get_index_stats
 from .retention import cleanup_old_logs, get_retention_stats
 from .jenkins.cli import jenkins_app
 from .version import __version__
@@ -462,6 +462,117 @@ def refresh(
 		raise typer.Exit(1)
 
 
+def _check_collector_liveness(collector_url: str) -> tuple[bool, str]:
+	"""Probe a collector URL for liveness. Returns (reachable, detail)."""
+	import urllib.request
+	import urllib.error
+	from .devlogs_client import _parse_collector_url
+
+	clean_url, token = _parse_collector_url(collector_url)
+	endpoint = clean_url.rstrip("/")
+	headers = {}
+	if token:
+		headers["Authorization"] = f"Bearer {token}"
+	req = urllib.request.Request(endpoint, method="HEAD", headers=headers)
+	try:
+		with urllib.request.urlopen(req, timeout=5):
+			pass
+		return True, "reachable"
+	except urllib.error.HTTPError as e:
+		# Any HTTP response means the server is up
+		return True, f"reachable (HTTP {e.code})"
+	except urllib.error.URLError as e:
+		return False, str(e.reason)
+	except Exception as e:
+		return False, f"{type(e).__name__}: {e}"
+
+
+def _print_index_stats(stats: dict):
+	"""Print OpenSearch index statistics."""
+	typer.echo(f"Total records:  {stats['total']:,}")
+	typer.echo(f"First entry:    {stats['first_entry'] or '(none)'}")
+	typer.echo(f"Last entry:     {stats['last_entry'] or '(none)'}")
+
+	counts = stats["counts_by_level"]
+	if counts:
+		typer.echo()
+		typer.echo("Records by level:")
+		for level_name in ("debug", "info", "warning", "error", "critical"):
+			if level_name in counts:
+				typer.echo(f"  {level_name + ':':12s} {counts[level_name]:>8,}")
+		for level_name in sorted(counts):
+			if level_name not in ("debug", "info", "warning", "error", "critical"):
+				typer.echo(f"  {level_name + ':':12s} {counts[level_name]:>8,}")
+
+
+@app.command()
+def status(
+	env: str = ENV_OPTION,
+	url: str = URL_OPTION,
+):
+	"""Show connection status and index statistics.
+
+	Detects the configured mode automatically:
+
+	  OpenSearch mode  — queries the index for record counts, timestamps,
+	                     and level breakdowns.
+
+	  Collector mode   — probes the write-only HTTP endpoint for liveness.
+	                     Collectors accept logs but cannot be queried, so
+	                     index statistics are not available.
+	"""
+	_apply_common_options(env, url)
+	cfg = load_config()
+
+	if cfg.url_mode == "collector":
+		typer.echo(f"Mode:           collector (write-only)")
+		typer.echo(f"Collector URL:  {cfg.collector_url}")
+
+		# Check for plugin
+		from .collector.plugins import get_plugin_for_url
+		plugin = get_plugin_for_url(cfg.collector_url, cfg)
+		if plugin:
+			typer.echo(f"Plugin:         {plugin.name}")
+			try:
+				plugin_status = plugin.check()
+				typer.echo(typer.style(f"Status:         {plugin_status}", fg=typer.colors.GREEN))
+			except Exception as e:
+				typer.echo(typer.style(f"Status:         unreachable ({e})", fg=typer.colors.RED))
+				raise typer.Exit(1)
+		else:
+			reachable, detail = _check_collector_liveness(cfg.collector_url)
+			if reachable:
+				typer.echo(typer.style(f"Status:         {detail}", fg=typer.colors.GREEN))
+			else:
+				typer.echo(typer.style(f"Status:         unreachable ({detail})", fg=typer.colors.RED))
+				raise typer.Exit(1)
+
+		typer.echo()
+		typer.echo(typer.style(
+			"Index statistics are not available in collector mode.\n"
+			"The collector is a write-only HTTP endpoint that accepts logs\n"
+			"but cannot be queried. Use an OpenSearch URL to see index stats.",
+			dim=True,
+		))
+		return
+
+	if cfg.url_mode == "none":
+		typer.echo(typer.style(
+			"Error: No OpenSearch or collector URL configured.\n"
+			"Set DEVLOGS_URL or DEVLOGS_OPENSEARCH_HOST.",
+			fg=typer.colors.RED,
+		), err=True)
+		raise typer.Exit(1)
+
+	# OpenSearch mode
+	client, cfg = require_opensearch()
+	typer.echo(f"Mode:           opensearch (read/write)")
+	typer.echo(f"Index:          {cfg.index}")
+
+	stats = get_index_stats(client, cfg.index)
+	_print_index_stats(stats)
+
+
 @app.command()
 def diagnose(
 	env: str = ENV_OPTION,

devlogs-2.3.0/src/devlogs/collector/loki_plugin.py

@@ -0,0 +1,182 @@
+# Loki output plugin for the devlogs collector
+#
+# Activated by setting DEVLOGS_FORWARD_URL=loki://host:3100 (or lokis:// for TLS).
+# Converts validated DevlogsRecord objects into Loki stream push payloads.
+#
+# Label strategy (low-cardinality only):
+#   application, component, level, area, environment
+#
+# Everything else (message, operation_id, fields, etc.) is stored as a JSON
+# log line and accessible via Loki's | json pipeline.
+
+import json
+import time
+import urllib.error
+import urllib.request
+from datetime import datetime, timezone
+from typing import Any, Dict, List
+from urllib.parse import urlparse
+
+from .errors import PluginError
+from .plugins import OutputPlugin, register_plugin
+from .schema import DevlogsRecord
+
+# Labels promoted to Loki stream labels (kept low-cardinality)
+_LOKI_LABEL_FIELDS = ("application", "component", "level", "area", "environment")
+
+
+def _record_to_ns(record: DevlogsRecord) -> int:
+    """Convert record timestamp to Unix nanoseconds for Loki."""
+    ts_str = record.timestamp or record.collected_ts
+    if not ts_str:
+        return int(time.time() * 1e9)
+    try:
+        # fromisoformat() doesn't accept 'Z' suffix in Python < 3.11
+        ts_clean = ts_str
+        if ts_clean.endswith("Z"):
+            ts_clean = ts_clean[:-1] + "+00:00"
+        dt = datetime.fromisoformat(ts_clean)
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+        return int(dt.timestamp() * 1e9)
+    except Exception:
+        return int(time.time() * 1e9)
+
+
+def _build_log_line(record: DevlogsRecord) -> str:
+    """Serialize the log record as a JSON log line for Loki storage."""
+    payload: Dict[str, Any] = {}
+    if record.message is not None:
+        payload["message"] = record.message
+    if record.operation_id is not None:
+        payload["operation_id"] = record.operation_id
+    payload["timestamp"] = record.timestamp
+    if record.collected_ts is not None:
+        payload["collected_ts"] = record.collected_ts
+    if record.version is not None:
+        payload["version"] = record.version
+    if record.fields:
+        payload["fields"] = record.fields
+    return json.dumps(payload, ensure_ascii=False)
+
+
+def _stream_key(record: DevlogsRecord) -> tuple:
+    """Return a hashable key representing this record's Loki stream labels."""
+    labels = []
+    if record.application:
+        labels.append(("application", record.application))
+    if record.component:
+        labels.append(("component", record.component))
+    if record.level:
+        labels.append(("level", record.level.lower()))
+    if record.area:
+        labels.append(("area", record.area))
+    if record.environment:
+        labels.append(("environment", record.environment))
+    return tuple(labels)
+
+
+class LokiOutputPlugin(OutputPlugin):
+    """Pushes log records to Grafana Loki via the HTTP push API.
+
+    URL schemes:
+        loki://host:port   — plain HTTP
+        lokis://host:port  — HTTPS
+    """
+
+    name = "loki"
+    schemes = ["loki", "lokis"]
+
+    def __init__(self, url: str, cfg: Any):
+        parsed = urlparse(url)
+        scheme = "https" if parsed.scheme == "lokis" else "http"
+        host = parsed.hostname or "localhost"
+        port = parsed.port or 3100
+        base = f"{scheme}://{host}:{port}"
+        self._push_url = f"{base}/loki/api/v1/push"
+        self._ready_url = f"{base}/ready"
+        self._display_url = url  # original url for display_info
+
+    def send(self, records: List[DevlogsRecord]) -> Dict[str, Any]:
+        """Push records to Loki, grouped into streams by label combination."""
+        streams: Dict[tuple, list] = {}
+        for record in records:
+            key = _stream_key(record)
+            if key not in streams:
+                streams[key] = []
+            ns = _record_to_ns(record)
+            line = _build_log_line(record)
+            streams[key].append([str(ns), line])
+
+        payload = {
+            "streams": [
+                {"stream": dict(key), "values": values}
+                for key, values in streams.items()
+            ]
+        }
+
+        body = json.dumps(payload).encode("utf-8")
+        req = urllib.request.Request(
+            self._push_url,
+            data=body,
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+
+        for attempt in range(3):
+            try:
+                with urllib.request.urlopen(req, timeout=10) as resp:
+                    # Loki returns 204 No Content on success
+                    if resp.status in (200, 204):
+                        return {"ingested": len(records)}
+                    raise PluginError(
+                        "UNEXPECTED_STATUS",
+                        f"Loki push returned unexpected status {resp.status}",
+                    )
+            except urllib.error.HTTPError as e:
+                if e.code in (429, 500, 502, 503, 504) and attempt < 2:
+                    time.sleep(0.5 * (2 ** attempt))
+                    continue
+                try:
+                    detail = e.read().decode("utf-8", errors="replace")[:300]
+                except Exception:
+                    detail = ""
+                raise PluginError(
+                    "HTTP_ERROR",
+                    f"Loki push failed with HTTP {e.code}: {detail}",
+                )
+            except urllib.error.URLError as e:
+                if attempt < 2:
+                    time.sleep(0.5 * (2 ** attempt))
+                    continue
+                raise PluginError(
+                    "SEND_FAILED",
+                    f"Failed to connect to Loki: {e.reason}",
+                )
+            except PluginError:
+                raise
+            except Exception as e:
+                if attempt < 2:
+                    time.sleep(0.5 * (2 ** attempt))
+                    continue
+                raise PluginError("SEND_FAILED", f"Loki send failed: {e}")
+
+        raise PluginError("SEND_FAILED", "Failed to push to Loki after retries")
+
+    def check(self) -> str:
+        """Verify Loki is reachable via its /ready endpoint."""
+        try:
+            req = urllib.request.Request(self._ready_url, method="GET")
+            with urllib.request.urlopen(req, timeout=5) as resp:
+                if resp.status == 200:
+                    return f"Loki: OK ({self._push_url})"
+                return f"Loki: /ready returned {resp.status}"
+        except Exception as e:
+            raise Exception(f"Loki not reachable at {self._ready_url}: {e}")
+
+    def display_info(self) -> str:
+        return f"Loki: {self._display_url}"
+
+
+# Self-register when this module is imported
+register_plugin(LokiOutputPlugin)

{devlogs-2.2.8 → devlogs-2.3.0}/src/devlogs/collector/schema.py

@@ -14,9 +14,12 @@
 #   - environment (string): Deployment environment (e.g., "development", "staging", "production")
 #   - version (string): Application version
 #
-# Collector-set fields (added during ingestion):
+# Collector-set fields (added during ingestion, always overwritten by collector):
 #   - collected_ts (string): ISO 8601 UTC timestamp when collector received the record
-#   - client_ip (string): IP address of the submitting client
+#   - client_ip (string): IP address of the submitting client as a bare IPv4 or IPv6
+#     address string (e.g. "192.168.1.5", "::1"). No port, no brackets, no CIDR suffix.
+#     Always set by the collector from the network connection; any value in the payload
+#     is discarded.
 #   - identity (object): Identity information resolved from auth token
 #     - mode: "anonymous" | "verified" | "passthrough"
 #     - For verified: id, name (optional), type (optional), tags (optional)
@@ -60,7 +63,8 @@ class DevlogsRecord:
         version: Application version (optional)
         fields: Arbitrary nested JSON for custom data (optional)
         collected_ts: Timestamp when collector received (set by collector)
-        client_ip: Submitting client IP address (set by collector)
+        client_ip: Submitting client IP address as a bare IPv4 or IPv6
+            string (set by collector, never accepted from payload)
         identity: Identity information resolved from auth (set by collector)
     """
 
@@ -329,9 +333,12 @@ def enrich_record(
 ) -> DevlogsRecord:
     """Enrich a record with collector metadata.
 
+    All three collector-set fields are unconditionally overwritten here;
+    any values from the payload are discarded.
+
     Args:
         record: The validated record
-        client_ip: IP address of the submitting client
+        client_ip: Bare IPv4/IPv6 address string from get_client_ip()
         identity: Identity object or dict from auth resolution
 
     Returns:

{devlogs-2.2.8 → devlogs-2.3.0}/src/devlogs/collector/server.py

@@ -5,10 +5,13 @@
 # - Ingest mode: write directly to OpenSearch
 
 import json
+import logging
 import platform
 from contextlib import asynccontextmanager
 from typing import Optional
 
+logger = logging.getLogger("devlogs.collector")
+
 from fastapi import FastAPI, Request, Response, HTTPException
 from fastapi.responses import JSONResponse
 from starlette.middleware.cors import CORSMiddleware
@@ -41,6 +44,13 @@ from .ingestor import ingest_records
 from .plugins import get_plugin_for_url
 from ..version import __version__
 
+# Register built-in output plugins by importing them (side-effect: register_plugin is called)
+try:
+    from . import loki_plugin as _loki_plugin  # noqa: F401
+except ImportError:
+    pass
+
+
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     """Emit a startup trace to the index so operators can see when the collector started."""
@@ -120,6 +130,9 @@ app.add_middleware(
 def get_client_ip(request: Request) -> str:
     """Extract client IP from request.
 
+    Returns a bare IPv4 or IPv6 address string (e.g. "192.168.1.5", "::1").
+    No port, no brackets, no CIDR suffix.
+
     Checks X-Forwarded-For header first (for proxied requests),
     then falls back to direct client connection.
     """
@@ -144,6 +157,8 @@ def get_client_ip(request: Request) -> str:
 @app.exception_handler(CollectorError)
 async def collector_error_handler(request: Request, exc: CollectorError):
     """Handle CollectorError exceptions with structured response."""
+    client_ip = get_client_ip(request)
+    logger.warning("%s %d %s: %s", client_ip, exc.status_code, exc.subcode, exc.message)
     return JSONResponse(
         status_code=exc.status_code,
         content=exc.to_dict(),
@@ -187,6 +202,8 @@ async def ingest_logs(request: Request):
 
     # Determine operating mode
     mode = cfg.get_collector_mode()
+    client_ip = get_client_ip(request)
+    logger.info("%s POST / (%d bytes)", client_ip, len(body))
 
     if mode == "forward":
         try:
@@ -223,8 +240,11 @@ async def _handle_forward_mode(request: Request, cfg, body: bytes) -> Response:
         timeout=cfg.opensearch_timeout,
     )
 
+    client_ip = get_client_ip(request)
+
     # If upstream returned 2xx, return 202
     if 200 <= status < 300:
+        logger.info("%s 202 forwarded", client_ip)
         return Response(
             status_code=202,
             content=json.dumps({"status": "accepted", "forwarded": True}),
@@ -232,6 +252,7 @@ async def _handle_forward_mode(request: Request, cfg, body: bytes) -> Response:
         )
     else:
         # This shouldn't happen - HTTPError should have been raised
+        logger.warning("%s %d forward failed", client_ip, status)
         return JSONResponse(
             status_code=status,
             content=response_body,
@@ -321,6 +342,8 @@ async def _handle_ingest_mode(request: Request, cfg, body: bytes) -> Response:
     index_map = parse_forward_index_map_kv(cfg.forward_index_map_kv)
 
     result = ingest_records(client, cfg.index, enriched_records, index_map)
+    client_ip = get_client_ip(request)
+    logger.info("%s 202 ingested %d record(s)", client_ip, result["ingested"])
 
     return Response(
         status_code=202,
@@ -353,11 +376,15 @@ async def _handle_plugin_mode(request: Request, cfg, body: bytes, plugin) -> Res
     if not isinstance(result, dict):
         result = {}
 
+    ingested = result.get("ingested", len(enriched_records))
+    client_ip = get_client_ip(request)
+    logger.info("%s 202 plugin '%s' ingested %d record(s)", client_ip, plugin.name, ingested)
+
     return Response(
         status_code=202,
         content=json.dumps({
             "status": "accepted",
-            "ingested": result.get("ingested", len(enriched_records)),
+            "ingested": ingested,
         }),
         media_type="application/json",
     )

devlogs-2.3.0/src/devlogs/loki/__init__.py

@@ -0,0 +1 @@
+# Loki query module for devlogs