devlogs 2.0.0__tar.gz → 2.0.2__tar.gz

{devlogs-2.0.0/src/devlogs.egg-info → devlogs-2.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devlogs
-Version: 2.0.0
+Version: 2.0.2
 Summary: Developer-focused logging library for Python with OpenSearch integration.
 Author-email: Dan Driscoll <dan@thedandriscoll.org>
 License: MIT License

{devlogs-2.0.0 → devlogs-2.0.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "devlogs"
-version = "2.0.0"
+version = "2.0.2"
 description = "Developer-focused logging library for Python with OpenSearch integration."
 requires-python = ">=3.11"
 readme = "README.md"

{devlogs-2.0.0 → devlogs-2.0.2}/src/devlogs/cli.py

@@ -20,7 +20,14 @@ from .opensearch.client import (
 	ConnectionFailedError,
 	DevlogsDisabledError,
 )
-from .opensearch.mappings import build_log_index_template, get_template_names
+from .opensearch.mappings import (
+	build_log_index_template,
+	get_template_names,
+	detect_schema_version,
+	get_schema_issues,
+	build_reindex_script,
+	SCHEMA_VERSION,
+)
 from .opensearch.queries import normalize_log_entries, search_logs, tail_logs, get_last_errors
 from .retention import cleanup_old_logs, get_retention_stats
 from .jenkins.cli import jenkins_app
@@ -184,17 +191,148 @@ def _write_codex_config(path: Path, python_path: str) -> str:
 	return "written"
 
 
+def _check_schema_compatibility(client, index: str) -> tuple[int | None, list[str]]:
+	"""Check index schema compatibility and return (version, issues)."""
+	try:
+		mapping = client.indices.get_mapping(index=index)
+		version = detect_schema_version(mapping)
+		issues = get_schema_issues(mapping) if version != SCHEMA_VERSION else []
+		return version, issues
+	except Exception:
+		return None, []
+
+
+def _perform_upgrade(client, cfg, source_index: str) -> bool:
+	"""Upgrade index to v2 schema by reindexing.
+
+	Returns True on success, False on failure.
+	"""
+	import uuid
+
+	target_index = f"{source_index}-v2-{uuid.uuid4().hex[:8]}"
+	template_body = build_log_index_template(cfg.index)
+
+	typer.echo(f"Creating new index '{target_index}' with v2 schema...")
+	try:
+		client.indices.create(index=target_index, body=template_body["template"])
+	except Exception as e:
+		typer.echo(typer.style(f"Error creating target index: {e}", fg=typer.colors.RED), err=True)
+		return False
+
+	typer.echo(f"Reindexing from '{source_index}' to '{target_index}'...")
+	typer.echo(typer.style("This may take a while for large indices...", dim=True))
+
+	reindex_body = {
+		"source": {"index": source_index},
+		"dest": {"index": target_index},
+		"script": {
+			"source": build_reindex_script(),
+			"lang": "painless",
+		},
+	}
+
+	try:
+		result = client.indices.reindex(body=reindex_body)
+		total = result.get("total", 0)
+		created = result.get("created", 0)
+		updated = result.get("updated", 0)
+		failures = result.get("failures", [])
+
+		if failures:
+			typer.echo(typer.style(f"Warning: {len(failures)} documents failed to reindex", fg=typer.colors.YELLOW))
+			for failure in failures[:3]:
+				typer.echo(f"  - {failure}", err=True)
+			if len(failures) > 3:
+				typer.echo(f"  ... and {len(failures) - 3} more", err=True)
+
+		typer.echo(f"Reindexed {total} documents ({created} created, {updated} updated)")
+	except Exception as e:
+		typer.echo(typer.style(f"Error during reindex: {e}", fg=typer.colors.RED), err=True)
+		typer.echo(f"The partial index '{target_index}' may need to be cleaned up manually.")
+		return False
+
+	# Delete old index and rename new one
+	typer.echo(f"Removing old index '{source_index}'...")
+	try:
+		client.indices.delete(index=source_index)
+	except Exception as e:
+		typer.echo(typer.style(f"Error deleting old index: {e}", fg=typer.colors.RED), err=True)
+		typer.echo(f"New index is available at '{target_index}'. Manual cleanup may be needed.")
+		return False
+
+	# Create alias or new index with original name pointing to data
+	typer.echo(f"Creating new index '{source_index}' with v2 schema...")
+	try:
+		client.indices.create(index=source_index, body=template_body["template"])
+	except Exception as e:
+		typer.echo(typer.style(f"Error creating new index: {e}", fg=typer.colors.RED), err=True)
+		typer.echo(f"Data is available at '{target_index}'.")
+		return False
+
+	# Reindex from temp to final
+	typer.echo(f"Moving data to '{source_index}'...")
+	reindex_final = {
+		"source": {"index": target_index},
+		"dest": {"index": source_index},
+	}
+	try:
+		client.indices.reindex(body=reindex_final)
+		client.indices.delete(index=target_index)
+	except Exception as e:
+		typer.echo(typer.style(f"Error finalizing: {e}", fg=typer.colors.RED), err=True)
+		typer.echo(f"Data may be split between '{source_index}' and '{target_index}'.")
+		return False
+
+	typer.echo(typer.style(f"Successfully upgraded '{source_index}' to v2 schema!", fg=typer.colors.GREEN))
+	return True
+
+
 @app.command()
 def init(
+	upgrade: bool = typer.Option(False, "--upgrade", help="Upgrade existing index to v2 schema if needed"),
 	env: str = ENV_OPTION,
 	url: str = URL_OPTION,
 ):
-	"""Initialize OpenSearch indices and templates (idempotent)."""
+	"""Initialize OpenSearch indices and templates (idempotent).
+
+	Checks existing index for v2 schema compatibility. Use --upgrade to
+	automatically migrate data from v1 to v2 schema.
+	"""
 	_apply_common_options(env, url)
 	client, cfg = require_opensearch(check_idx=False)
+
+	# Check existing index schema
+	index_exists = client.indices.exists(index=cfg.index)
+	if index_exists:
+		version, issues = _check_schema_compatibility(client, cfg.index)
+		if version is not None:
+			typer.echo(f"Index '{cfg.index}' exists with schema v{version}")
+			if version == SCHEMA_VERSION:
+				typer.echo(typer.style("Schema is v2-compatible.", fg=typer.colors.GREEN))
+			else:
+				typer.echo(typer.style(f"Schema needs upgrade to v{SCHEMA_VERSION}.", fg=typer.colors.YELLOW))
+				if issues:
+					typer.echo("Issues found:")
+					for issue in issues:
+						typer.echo(f"  - {issue}")
+
+				if upgrade:
+					typer.echo("")
+					if not _perform_upgrade(client, cfg, cfg.index):
+						raise typer.Exit(1)
+				else:
+					typer.echo("")
+					typer.echo("Run with --upgrade to migrate data to v2 schema.")
+					typer.echo(typer.style(
+						"Warning: Upgrade will reindex all data. Back up your index first.",
+						fg=typer.colors.YELLOW,
+					))
+					raise typer.Exit(1)
+
 	# Create or update index templates
 	template_body = build_log_index_template(cfg.index)
 	template_name, legacy_template_name = get_template_names(cfg.index)
+
 	# Remove any conflicting templates before creating a new one
 	names_to_remove = {template_name, legacy_template_name}
 	names_to_remove.update(OLD_TEMPLATE_NAMES)
@@ -210,10 +348,12 @@ def init(
 					err=True,
 				)
 	client.indices.put_index_template(name=template_name, body=template_body)
+
 	# Create initial index with explicit mappings if it doesn't exist
-	if not client.indices.exists(index=cfg.index):
+	if not index_exists:
 		client.indices.create(index=cfg.index, body=template_body["template"])
-		typer.echo(f"Created index '{cfg.index}'.")
+		typer.echo(f"Created index '{cfg.index}' with v{SCHEMA_VERSION} schema.")
+
 	typer.echo("OpenSearch indices and templates initialized.")
 
 
@@ -651,7 +791,7 @@ def tail(
 				entry_area = doc.get("area") or ""
 				entry_operation = doc.get("operation_id") or ""
 				message = doc.get("message") or ""
-				features = _format_features(doc.get("features"))
+				features = _format_features(doc.get("fields"))
 				if features:
 					typer.echo(f"{timestamp} {entry_level} {entry_area} {entry_operation} {features} {message}")
 				else:
@@ -762,7 +902,7 @@ def search(
 			entry_area = doc.get("area") or ""
 			entry_operation = doc.get("operation_id") or ""
 			message = doc.get("message") or ""
-			features = _format_features(doc.get("features"))
+			features = _format_features(doc.get("fields"))
 			if features:
 				typer.echo(f"{timestamp} {entry_level} {entry_area} {entry_operation} {features} {message}")
 			else:
@@ -838,7 +978,7 @@ def last_error(
 		entry_area = doc.get("area") or ""
 		entry_operation = doc.get("operation_id") or ""
 		message = doc.get("message") or ""
-		features = _format_features(doc.get("features"))
+		features = _format_features(doc.get("fields"))
 		if features:
 			typer.echo(f"{timestamp} {entry_level} {entry_area} {entry_operation} {features} {message}")
 		else:

{devlogs-2.0.0 → devlogs-2.0.2}/src/devlogs/devlogs_client.py

@@ -8,18 +8,104 @@ import urllib.request
 import urllib.error
 from dataclasses import dataclass, field
 from datetime import datetime, timezone
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
+from urllib.parse import urlparse, unquote, urlunparse
+
+
+def _parse_collector_url(url: str) -> Tuple[str, Optional[str]]:
+    """Parse a URL and extract auth token if it's a collector URL.
+
+    Distinguishes between OpenSearch URLs and collector URLs:
+    - OpenSearch URL: has BOTH username AND password - keep credentials in URL
+    - Collector URL: has only token in username position - extract for Bearer auth
+
+    Collector URL format: http://token@host:port
+    OpenSearch URL format: http://user:password@host:port
+
+    Args:
+        url: The URL, optionally with credentials in userinfo
+
+    Returns:
+        Tuple of (url, token):
+        - For OpenSearch URLs (user:pass): returns original URL, None
+        - For collector URLs (token only): returns clean URL without userinfo, token
+        - For plain URLs: returns original URL, None
+    """
+    if not url:
+        return url, None
+
+    parsed = urlparse(url)
+
+    # If no userinfo, return as-is
+    if not parsed.username and not parsed.password:
+        return url, None
+
+    # OpenSearch URL: has BOTH username AND password
+    # Keep the URL as-is with credentials, no Bearer token
+    if parsed.username and parsed.password:
+        return url, None
+
+    # Collector URL: token in username position only (no password)
+    # Extract token and strip userinfo from URL
+    token = unquote(parsed.username) if parsed.username else None
+
+    # Rebuild URL without userinfo
+    # netloc without userinfo is just host:port
+    if parsed.port:
+        netloc = f"{parsed.hostname}:{parsed.port}"
+    else:
+        netloc = parsed.hostname or ""
+
+    clean_url = urlunparse((
+        parsed.scheme,
+        netloc,
+        parsed.path,
+        parsed.params,
+        parsed.query,
+        parsed.fragment,
+    ))
+
+    return clean_url, token
 
 
 @dataclass
 class DevlogsClient:
-    """Client for sending logs to a devlogs collector.
+    """Client for sending logs to a devlogs collector or OpenSearch.
+
+    URL Types:
+        This client distinguishes between collector URLs and OpenSearch URLs:
+
+        Collector URL (token in username position):
+            http://dl1_myapp_secret@localhost:8080
+            - Token is extracted and sent as Bearer auth header
+            - Userinfo is stripped from the request URL
+
+        OpenSearch URL (both username AND password):
+            https://admin:password@opensearch.example.com:9200
+            - Credentials remain in the URL for HTTP Basic auth
+            - No Bearer token is used
 
     Usage:
+        # Collector URL with token:
+        client = DevlogsClient(
+            collector_url="http://dl1_myapp_secret@localhost:8080",
+            application="my-app",
+            component="api-server",
+        )
+
+        # OpenSearch URL with credentials:
+        client = DevlogsClient(
+            collector_url="https://admin:password@opensearch.example.com:9200",
+            application="my-app",
+            component="api-server",
+        )
+
+        # Or with explicit auth_token parameter:
         client = DevlogsClient(
             collector_url="http://localhost:8080",
             application="my-app",
             component="api-server",
+            auth_token="dl1_myapp_secret",
         )
 
         # Send a single log
@@ -34,6 +120,8 @@ class DevlogsClient:
             {"message": "Event 1", "level": "info"},
             {"message": "Event 2", "level": "warning"},
         ])
+
+    If both URL token and auth_token parameter are provided, auth_token takes precedence.
     """
 
     collector_url: str
@@ -44,16 +132,27 @@ class DevlogsClient:
     auth_token: Optional[str] = None
     timeout: int = 30
 
+    # Internal fields set by __post_init__
+    _clean_url: str = field(default="", init=False, repr=False)
+    _resolved_token: Optional[str] = field(default=None, init=False, repr=False)
+
+    def __post_init__(self):
+        """Parse collector URL and extract token if present."""
+        clean_url, url_token = _parse_collector_url(self.collector_url)
+        self._clean_url = clean_url
+        # Explicit auth_token parameter takes precedence over URL token
+        self._resolved_token = self.auth_token if self.auth_token else url_token
+
     def _get_endpoint(self) -> str:
         """Get the collector endpoint URL."""
-        base = self.collector_url.rstrip("/")
+        base = self._clean_url.rstrip("/")
         return f"{base}/v1/logs"
 
     def _get_headers(self) -> Dict[str, str]:
         """Get request headers."""
         headers = {"Content-Type": "application/json"}
-        if self.auth_token:
-            headers["Authorization"] = f"Bearer {self.auth_token}"
+        if self._resolved_token:
+            headers["Authorization"] = f"Bearer {self._resolved_token}"
         return headers
 
     def _now(self) -> str:
@@ -209,15 +308,17 @@ def create_client(
     version: Optional[str] = None,
     auth_token: Optional[str] = None,
 ) -> DevlogsClient:
-    """Create an Devlogs client.
+    """Create a Devlogs client.
 
     Args:
-        collector_url: The collector endpoint URL (DEVLOGS_URL)
+        collector_url: The endpoint URL. URL type is auto-detected:
+            - Collector URL: http://token@host:port (token becomes Bearer auth)
+            - OpenSearch URL: http://user:pass@host:port (credentials kept in URL)
         application: Application name
         component: Component name within the application
         environment: Deployment environment (optional)
         version: Application version (optional)
-        auth_token: Bearer token for authentication (optional)
+        auth_token: Bearer token for authentication (optional, overrides URL token)
 
     Returns:
         Configured DevlogsClient instance
@@ -248,7 +349,9 @@ def emit_log(
     For repeated logging, use create_client() instead.
 
     Args:
-        collector_url: The collector endpoint URL
+        collector_url: The endpoint URL. URL type is auto-detected:
+            - Collector URL: http://token@host:port (token becomes Bearer auth)
+            - OpenSearch URL: http://user:pass@host:port (credentials kept in URL)
         application: Application name
         component: Component name
         message: Log message
@@ -256,7 +359,7 @@ def emit_log(
         fields: Custom fields
         environment: Deployment environment
         version: Application version
-        auth_token: Bearer token
+        auth_token: Bearer token (optional, overrides URL token)
 
     Returns:
         True if accepted, False on error

{devlogs-2.0.0 → devlogs-2.0.2}/src/devlogs/handler.py

@@ -169,24 +169,20 @@ class DevlogsHandler(logging.Handler):
 		if operation_id:
 			doc["operation_id"] = operation_id
 
-		# Custom fields (renamed from 'features')
+		# Custom fields
 		fields = _extract_features(record)
 		if fields:
 			doc["fields"] = fields
 
-		# Source location info (useful for debugging)
-		doc["source"] = {
-			"logger": record.name,
-			"pathname": record.pathname,
-			"lineno": record.lineno,
-			"funcName": record.funcName,
-		}
+		# Source location info (flat schema to match mappings)
+		doc["logger"] = record.name
+		doc["pathname"] = record.pathname
+		doc["lineno"] = record.lineno
+		doc["funcname"] = record.funcName
 
-		# Process/thread info
-		doc["process"] = {
-			"id": record.process,
-			"thread": record.thread,
-		}
+		# Process/thread info (flat schema to match mappings)
+		doc["process"] = record.process
+		doc["thread"] = record.thread
 
 		# Exception info if present
 		exc_text = getattr(record, "exc_text", None)

{devlogs-2.0.0 → devlogs-2.0.2}/src/devlogs/opensearch/client.py

@@ -263,6 +263,14 @@ class _IndicesClient:
 		"""Refresh an index to make recent changes searchable."""
 		return self._client._request("POST", f"/{index}/_refresh")
 
+	def get_mapping(self, index):
+		"""Get index mapping."""
+		return self._client._request("GET", f"/{index}/_mapping")
+
+	def reindex(self, body):
+		"""Reindex documents from one index to another."""
+		return self._client._request("POST", "/_reindex", body)
+
 
 def get_opensearch_client():
 	cfg = load_config()

devlogs-2.0.2/src/devlogs/opensearch/mappings.py

@@ -0,0 +1,199 @@
+# OpenSearch index templates and mappings
+
+from typing import Optional
+
+
+# Current schema version
+SCHEMA_VERSION = 2
+
+# V2 required fields (flat schema)
+V2_REQUIRED_FIELDS = {"logger", "funcname", "fields"}
+
+# V1 fields that indicate old schema
+V1_INDICATOR_FIELDS = {"logger_name", "features", "funcName"}
+
+
+def detect_schema_version(mapping: dict) -> Optional[int]:
+	"""Detect schema version from index mapping.
+
+	Returns:
+		2 if v2-compatible (flat schema with logger, funcname, fields)
+		1 if v1 schema (nested source/process, logger_name, features)
+		None if unknown/empty
+	"""
+	if not mapping:
+		return None
+
+	# Extract properties from mapping (handle different response formats)
+	properties = mapping.get("properties", {})
+	if not properties:
+		# Try nested format from get_mapping response
+		for index_data in mapping.values():
+			if isinstance(index_data, dict):
+				properties = index_data.get("mappings", {}).get("properties", {})
+				if properties:
+					break
+
+	if not properties:
+		return None
+
+	field_names = set(properties.keys())
+
+	# Check for v2 indicators
+	has_v2_fields = V2_REQUIRED_FIELDS.issubset(field_names)
+
+	# Check for v1 indicators
+	has_v1_fields = bool(V1_INDICATOR_FIELDS & field_names)
+
+	# Check if process is an object (v1) vs integer (v2)
+	process_mapping = properties.get("process", {})
+	process_is_object = process_mapping.get("type") == "object" or "properties" in process_mapping
+
+	if has_v2_fields and not has_v1_fields and not process_is_object:
+		return 2
+	elif has_v1_fields or process_is_object:
+		return 1
+
+	# If we have some standard fields but can't determine version, assume v1
+	if field_names & {"timestamp", "level", "message"}:
+		return 1
+
+	return None
+
+
+def get_schema_issues(mapping: dict) -> list[str]:
+	"""Get list of schema compatibility issues.
+
+	Returns list of human-readable issues that need to be fixed for v2 compatibility.
+	"""
+	issues = []
+
+	properties = mapping.get("properties", {})
+	if not properties:
+		for index_data in mapping.values():
+			if isinstance(index_data, dict):
+				properties = index_data.get("mappings", {}).get("properties", {})
+				if properties:
+					break
+
+	if not properties:
+		return ["No mapping properties found"]
+
+	field_names = set(properties.keys())
+
+	# Check for old field names
+	if "logger_name" in field_names:
+		issues.append("Has 'logger_name' field (v2 uses 'logger')")
+	if "funcName" in field_names:
+		issues.append("Has 'funcName' field (v2 uses 'funcname')")
+	if "features" in field_names:
+		issues.append("Has 'features' field (v2 uses 'fields')")
+
+	# Check process type
+	process_mapping = properties.get("process", {})
+	if process_mapping.get("type") == "object" or "properties" in process_mapping:
+		issues.append("'process' is an object (v2 expects integer)")
+
+	# Check for nested source object
+	if "source" in field_names:
+		source_mapping = properties.get("source", {})
+		if source_mapping.get("type") == "object" or "properties" in source_mapping:
+			issues.append("Has nested 'source' object (v2 uses flat fields)")
+
+	# Check for missing v2 fields
+	for field in V2_REQUIRED_FIELDS:
+		if field not in field_names:
+			issues.append(f"Missing '{field}' field")
+
+	return issues
+
+
+def build_reindex_script() -> str:
+	"""Build Painless script to transform v1 documents to v2 schema."""
+	return """
+		// Transform logger_name to logger
+		if (ctx._source.containsKey('logger_name')) {
+			ctx._source.logger = ctx._source.remove('logger_name');
+		}
+		// Transform source.logger to logger (if nested)
+		if (ctx._source.containsKey('source') && ctx._source.source instanceof Map) {
+			if (ctx._source.source.containsKey('logger')) {
+				ctx._source.logger = ctx._source.source.logger;
+			}
+			if (ctx._source.source.containsKey('pathname')) {
+				ctx._source.pathname = ctx._source.source.pathname;
+			}
+			if (ctx._source.source.containsKey('lineno')) {
+				ctx._source.lineno = ctx._source.source.lineno;
+			}
+			if (ctx._source.source.containsKey('funcName')) {
+				ctx._source.funcname = ctx._source.source.funcName;
+			}
+			ctx._source.remove('source');
+		}
+		// Transform funcName to funcname
+		if (ctx._source.containsKey('funcName')) {
+			ctx._source.funcname = ctx._source.remove('funcName');
+		}
+		// Transform features to fields
+		if (ctx._source.containsKey('features')) {
+			ctx._source.fields = ctx._source.remove('features');
+		}
+		// Transform nested process object to flat fields
+		if (ctx._source.containsKey('process') && ctx._source.process instanceof Map) {
+			def proc = ctx._source.process;
+			if (proc.containsKey('id')) {
+				ctx._source.process = proc.id;
+			}
+			if (proc.containsKey('thread')) {
+				ctx._source.thread = proc.thread;
+			}
+		}
+	""".strip()
+
+
+def build_log_index_template(index_name: str) -> dict:
+	"""Return the composable index template for the exact index name."""
+	base_template = {
+		"index_patterns": [index_name],
+		"priority": 100,
+		"template": {
+			"settings": {"number_of_shards": 1, "number_of_replicas": 0},
+			"mappings": {
+				"properties": {
+					# Core log entry fields (flat schema)
+					"doc_type": {"type": "keyword"},  # Always "log_entry"
+					"timestamp": {"type": "date"},
+					"level": {"type": "keyword"},
+					"levelno": {"type": "integer"},
+					"logger": {"type": "keyword"},
+					"message": {"type": "text"},
+					"area": {"type": "keyword"},
+					"operation_id": {"type": "keyword"},
+					"pathname": {"type": "keyword"},
+					"lineno": {"type": "integer"},
+					"funcname": {"type": "keyword"},
+					"thread": {"type": "long"},
+					"process": {"type": "integer"},
+					"exception": {"type": "text"},
+					"fields": {"type": "object", "dynamic": True},
+				}
+			}
+		}
+	}
+	return base_template
+
+
+def build_legacy_log_template(index_name: str) -> dict:
+	"""Return the legacy template payload for clusters without composable templates."""
+	template = build_log_index_template(index_name)
+	return {
+		"index_patterns": template["index_patterns"],
+		"settings": template["template"]["settings"],
+		"mappings": template["template"]["mappings"],
+	}
+
+
+def get_template_names(index_name: str) -> tuple[str, str]:
+	"""Return deterministic template names based on the index name."""
+	return (f"{index_name}-template", f"{index_name}-legacy-template")