devlogs 2.0.2__tar.gz → 2.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. {devlogs-2.0.2/src/devlogs.egg-info → devlogs-2.1.0}/PKG-INFO +1 -1
  2. {devlogs-2.0.2 → devlogs-2.1.0}/pyproject.toml +1 -1
  3. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/cli.py +32 -159
  4. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/collector/cli.py +2 -0
  5. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/collector/server.py +66 -1
  6. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/config.py +201 -5
  7. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/demo.py +32 -19
  8. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/devlogs_client.py +21 -5
  9. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/handler.py +44 -4
  10. devlogs-2.1.0/src/devlogs/jenkins/cli.py +102 -0
  11. devlogs-2.1.0/src/devlogs/version.py +3 -0
  12. {devlogs-2.0.2 → devlogs-2.1.0/src/devlogs.egg-info}/PKG-INFO +1 -1
  13. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs.egg-info/SOURCES.txt +2 -0
  14. {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_cli.py +16 -26
  15. {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_config.py +8 -26
  16. {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_mcp_server.py +22 -21
  17. devlogs-2.1.0/tests/test_url_parsing.py +239 -0
  18. devlogs-2.0.2/src/devlogs/jenkins/cli.py +0 -257
  19. {devlogs-2.0.2 → devlogs-2.1.0}/LICENSE +0 -0
  20. {devlogs-2.0.2 → devlogs-2.1.0}/MANIFEST.in +0 -0
  21. {devlogs-2.0.2 → devlogs-2.1.0}/README.md +0 -0
  22. {devlogs-2.0.2 → devlogs-2.1.0}/setup.cfg +0 -0
  23. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/__init__.py +0 -0
  24. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/__main__.py +0 -0
  25. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/build_info.py +0 -0
  26. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/collector/__init__.py +0 -0
  27. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/collector/auth.py +0 -0
  28. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/collector/errors.py +0 -0
  29. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/collector/forwarder.py +0 -0
  30. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/collector/ingestor.py +0 -0
  31. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/collector/schema.py +0 -0
  32. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/context.py +0 -0
  33. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/formatting.py +0 -0
  34. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/jenkins/__init__.py +0 -0
  35. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/jenkins/core.py +0 -0
  36. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/levels.py +0 -0
  37. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/mcp/__init__.py +0 -0
  38. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/mcp/server.py +0 -0
  39. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/opensearch/__init__.py +0 -0
  40. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/opensearch/client.py +0 -0
  41. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/opensearch/indexing.py +0 -0
  42. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/opensearch/mappings.py +0 -0
  43. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/opensearch/queries.py +0 -0
  44. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/retention.py +0 -0
  45. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/scrub.py +0 -0
  46. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/time_utils.py +0 -0
  47. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/web/__init__.py +0 -0
  48. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/web/server.py +0 -0
  49. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/web/static/devlogs.css +0 -0
  50. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/web/static/devlogs.js +0 -0
  51. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/web/static/index.html +0 -0
  52. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/wrapper.py +0 -0
  53. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs.egg-info/dependency_links.txt +0 -0
  54. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs.egg-info/entry_points.txt +0 -0
  55. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs.egg-info/requires.txt +0 -0
  56. {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs.egg-info/top_level.txt +0 -0
  57. {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_build_info.py +0 -0
  58. {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_collector_auth.py +0 -0
  59. {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_collector_config.py +0 -0
  60. {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_collector_schema.py +0 -0
  61. {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_collector_server.py +0 -0
  62. {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_context.py +0 -0
  63. {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_devlogs_client.py +0 -0
  64. {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_formatting.py +0 -0
  65. {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_handler.py +0 -0
  66. {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_indexing.py +0 -0
  67. {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_levels.py +0 -0
  68. {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_mappings.py +0 -0
  69. {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_opensearch_client.py +0 -0
  70. {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_opensearch_queries.py +0 -0
  71. {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_retention.py +0 -0
  72. {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_scrub.py +0 -0
  73. {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_time_utils.py +0 -0
  74. {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_web.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: devlogs
3
- Version: 2.0.2
3
+ Version: 2.1.0
4
4
  Summary: Developer-focused logging library for Python with OpenSearch integration.
5
5
  Author-email: Dan Driscoll <dan@thedandriscoll.org>
6
6
  License: MIT License
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "devlogs"
7
- version = "2.0.2"
7
+ version = "2.1.0"
8
8
  description = "Developer-focused logging library for Python with OpenSearch integration."
9
9
  requires-python = ">=3.11"
10
10
  readme = "README.md"
@@ -10,7 +10,7 @@ import click
10
10
  import typer
11
11
  from pathlib import Path
12
12
 
13
- from .config import load_config, set_dotenv_path, set_url, URLParseError, _parse_opensearch_url
13
+ from .config import load_config, set_dotenv_path, set_url, URLParseError, _parse_opensearch_url, parse_url, CollectorURLConfig
14
14
  from .formatting import format_timestamp
15
15
  from .opensearch.client import (
16
16
  get_opensearch_client,
@@ -39,7 +39,7 @@ OLD_TEMPLATE_NAMES = ("devlogs-template", "devlogs-logs-template")
39
39
 
40
40
  # Common options for commands - these can be placed anywhere in the command line
41
41
  ENV_OPTION = typer.Option(None, "--env", help="Path to .env file to load")
42
- URL_OPTION = typer.Option(None, "--url", help="OpenSearch URL (e.g., https://user:pass@host:port/index)")
42
+ URL_OPTION = typer.Option(None, "--url", help="URL (e.g., opensearchs://user:pass@host:port/index or https://TOKEN@host:port/path)")
43
43
 
44
44
 
45
45
  def _apply_common_options(env: str = None, url: str = None):
@@ -61,11 +61,17 @@ def main_callback(
61
61
  _apply_common_options(env, url)
62
62
 
63
63
 
64
+ _HIDDEN_FIELDS = {"run_id", "build_number", "build_url", "job", "node_id", "seq"}
65
+
66
+
64
67
  def _format_features(features):
65
68
  if not features:
66
69
  return ""
67
70
  if isinstance(features, dict):
68
- items = sorted(features.items(), key=lambda item: str(item[0]))
71
+ items = sorted(
72
+ ((k, v) for k, v in features.items() if k not in _HIDDEN_FIELDS),
73
+ key=lambda item: str(item[0]),
74
+ )
69
75
  parts = []
70
76
  for key, value in items:
71
77
  key_text = str(key)
@@ -1166,7 +1172,13 @@ def demo(
1166
1172
  """Generate demo logs to illustrate devlogs capabilities."""
1167
1173
  _apply_common_options(env, url)
1168
1174
  from .demo import run_demo
1169
- run_demo(duration, count, require_opensearch)
1175
+
1176
+ cfg = load_config()
1177
+ if cfg.url_mode == "collector":
1178
+ from .devlogs_client import DevlogsClient
1179
+ run_demo(duration, count, require_opensearch=None, collector_url=cfg.collector_url)
1180
+ else:
1181
+ run_demo(duration, count, require_opensearch=require_opensearch)
1170
1182
 
1171
1183
 
1172
1184
  @app.command()
@@ -1181,8 +1193,20 @@ def serve(
1181
1193
 
1182
1194
 
1183
1195
  def _build_opensearch_url(scheme: str, host: str, port: int, user: str, password: str, index: str) -> str:
1184
- """Build an OpenSearch URL from components, URL-encoding credentials."""
1196
+ """Build an OpenSearch URL from components, URL-encoding credentials.
1197
+
1198
+ Uses the opensearchs:// scheme (TLS) or opensearch:// (non-TLS).
1199
+ The scheme parameter accepts 'https'/'http' (mapped to opensearch schemes)
1200
+ or 'opensearchs'/'opensearch' directly.
1201
+ """
1185
1202
  from urllib.parse import quote
1203
+ # Map legacy schemes to opensearch:// variants
1204
+ if scheme == "https":
1205
+ url_scheme = "opensearchs"
1206
+ elif scheme == "http":
1207
+ url_scheme = "opensearch"
1208
+ else:
1209
+ url_scheme = scheme
1186
1210
  # URL-encode username and password to handle special characters
1187
1211
  encoded_user = quote(user, safe="") if user else ""
1188
1212
  encoded_pass = quote(password, safe="") if password else ""
@@ -1193,7 +1217,7 @@ def _build_opensearch_url(scheme: str, host: str, port: int, user: str, password
1193
1217
  else:
1194
1218
  auth = ""
1195
1219
  path = f"/{index}" if index else ""
1196
- return f"{scheme}://{auth}{host}:{port}{path}"
1220
+ return f"{url_scheme}://{auth}{host}:{port}{path}"
1197
1221
 
1198
1222
 
1199
1223
  def _format_env_output(scheme: str, host: str, port: int, user: str, password: str, index: str) -> str:
@@ -1211,155 +1235,6 @@ def _format_env_output(scheme: str, host: str, port: int, user: str, password: s
1211
1235
  return "\n".join(lines)
1212
1236
 
1213
1237
 
1214
- @app.command()
1215
- def initjenkins(
1216
- jenkinsfile: str = typer.Argument("Jenkinsfile", help="Path to Jenkinsfile to modify"),
1217
- credential_id: str = typer.Option("devlogs-opensearch-url", "--credential-id", "-c", help="Jenkins credential ID to use"),
1218
- env: str = ENV_OPTION,
1219
- url: str = URL_OPTION,
1220
- ):
1221
- """Add devlogs configuration to an existing Jenkinsfile.
1222
-
1223
- This command modifies an existing Jenkinsfile to add an options block
1224
- with the devlogs pipeline step configured to use a Jenkins credential.
1225
-
1226
- After running this command, you need to create a Jenkins credential
1227
- of type "Secret text" with the OpenSearch URL.
1228
-
1229
- Examples:
1230
- devlogs initjenkins # Modify ./Jenkinsfile
1231
- devlogs initjenkins path/to/Jenkinsfile # Modify specific file
1232
- devlogs initjenkins --credential-id my-cred # Use custom credential ID
1233
- """
1234
- import re
1235
-
1236
- _apply_common_options(env, url)
1237
-
1238
- jenkinsfile_path = Path(jenkinsfile)
1239
- if not jenkinsfile_path.is_file():
1240
- typer.echo(typer.style(f"Error: Jenkinsfile not found: {jenkinsfile_path}", fg=typer.colors.RED), err=True)
1241
- raise typer.Exit(1)
1242
-
1243
- content = jenkinsfile_path.read_text(encoding="utf-8")
1244
-
1245
- # Check if this looks like a declarative pipeline
1246
- if "pipeline" not in content:
1247
- typer.echo(typer.style("Error: File does not appear to be a declarative Jenkins pipeline.", fg=typer.colors.RED), err=True)
1248
- typer.echo("This command only supports declarative pipelines with a 'pipeline { }' block.", err=True)
1249
- raise typer.Exit(1)
1250
-
1251
- # Check if devlogs is already configured
1252
- if "devlogs(" in content:
1253
- typer.echo(typer.style("Warning: Jenkinsfile already appears to have devlogs configuration.", fg=typer.colors.YELLOW))
1254
- typer.echo("Review the file manually to ensure correct configuration.")
1255
- raise typer.Exit(0)
1256
-
1257
- options_line = f" devlogs(credentialsId: '{credential_id}')"
1258
-
1259
- modified = False
1260
- lines = content.split("\n")
1261
- result_lines = []
1262
- i = 0
1263
-
1264
- # Track brace depth and whether we're inside pipeline block
1265
- in_pipeline = False
1266
- pipeline_brace_depth = 0
1267
- added_options = False
1268
-
1269
- while i < len(lines):
1270
- line = lines[i]
1271
- result_lines.append(line)
1272
-
1273
- # Detect entering pipeline block
1274
- if not in_pipeline and re.match(r'^\s*pipeline\s*\{', line):
1275
- in_pipeline = True
1276
- pipeline_brace_depth = 1
1277
- i += 1
1278
- continue
1279
-
1280
- if in_pipeline:
1281
- # Count braces to track depth
1282
- pipeline_brace_depth += line.count('{') - line.count('}')
1283
-
1284
- # Check for existing options block and add our line
1285
- if not added_options and re.match(r'^\s*options\s*\{', line):
1286
- result_lines.append(options_line)
1287
- added_options = True
1288
- modified = True
1289
-
1290
- # If we hit stages and haven't added options, add it before stages
1291
- if re.match(r'^\s*stages\s*\{', line) and not added_options:
1292
- # Insert before the stages line
1293
- result_lines.pop() # Remove the stages line we just added
1294
-
1295
- result_lines.append("")
1296
- result_lines.append(" options {")
1297
- result_lines.append(options_line)
1298
- result_lines.append(" }")
1299
- added_options = True
1300
- modified = True
1301
-
1302
- result_lines.append("")
1303
- result_lines.append(line) # Re-add the stages line
1304
-
1305
- # Exit pipeline tracking when we close the pipeline block
1306
- if pipeline_brace_depth == 0:
1307
- in_pipeline = False
1308
-
1309
- i += 1
1310
-
1311
- if not modified:
1312
- typer.echo(typer.style("Error: Could not find a suitable location to add devlogs configuration.", fg=typer.colors.RED), err=True)
1313
- typer.echo("Ensure the Jenkinsfile has a 'pipeline { stages { } }' structure.", err=True)
1314
- raise typer.Exit(1)
1315
-
1316
- # Write the modified file
1317
- new_content = "\n".join(result_lines)
1318
- jenkinsfile_path.write_text(new_content, encoding="utf-8")
1319
-
1320
- typer.echo(typer.style(f"Modified {jenkinsfile_path}", fg=typer.colors.GREEN))
1321
- typer.echo()
1322
- typer.echo("Added:")
1323
- typer.echo(f" - Options: devlogs(credentialsId: '{credential_id}')")
1324
-
1325
- # Print setup instructions
1326
- typer.echo()
1327
- typer.echo(typer.style("Next steps - Create Jenkins credential:", fg=typer.colors.CYAN, bold=True))
1328
- typer.echo("=" * 60)
1329
- typer.echo()
1330
- typer.echo("1. Go to Jenkins > Manage Jenkins > Credentials")
1331
- typer.echo("2. Select the appropriate domain (e.g., Global)")
1332
- typer.echo("3. Click 'Add Credentials'")
1333
- typer.echo("4. Configure:")
1334
- typer.echo(f" - Kind: Secret text")
1335
- typer.echo(f" - ID: {credential_id}")
1336
- typer.echo(" - Secret: <your OpenSearch URL>")
1337
- typer.echo()
1338
-
1339
- # Try to load config and show the URL value
1340
- try:
1341
- cfg = load_config()
1342
- if cfg.enabled:
1343
- # Build the URL from config
1344
- credential_url = _build_opensearch_url(
1345
- cfg.opensearch_scheme,
1346
- cfg.opensearch_host,
1347
- cfg.opensearch_port,
1348
- cfg.opensearch_user,
1349
- cfg.opensearch_pass,
1350
- cfg.index,
1351
- )
1352
- typer.echo(typer.style("Credential value (from your .env/environment):", fg=typer.colors.GREEN, bold=True))
1353
- typer.echo("-" * 60)
1354
- typer.echo(credential_url)
1355
- typer.echo("-" * 60)
1356
- else:
1357
- typer.echo("Tip: Set up a .env file with DEVLOGS_OPENSEARCH_URL to see the exact value here.")
1358
- typer.echo(" Run 'devlogs mkurl' to interactively build the URL.")
1359
- except Exception:
1360
- typer.echo("Tip: Run 'devlogs mkurl' to interactively build the OpenSearch URL.")
1361
-
1362
-
1363
1238
  @app.command()
1364
1239
  def mkurl():
1365
1240
  """Interactively create an OpenSearch URL and show .env formats.
@@ -1410,10 +1285,8 @@ def mkurl():
1410
1285
  else:
1411
1286
  # Prompt for each component
1412
1287
  typer.echo()
1413
- scheme = typer.prompt("Scheme (http/https)", default="https")
1414
- if scheme not in ("http", "https"):
1415
- typer.echo(typer.style("Error: Scheme must be 'http' or 'https'.", fg=typer.colors.RED), err=True)
1416
- raise typer.Exit(1)
1288
+ use_tls = typer.prompt("Use TLS? (yes/no)", default="yes")
1289
+ scheme = "https" if use_tls.lower() in ("yes", "y") else "http"
1417
1290
  host = typer.prompt("Host", default="localhost")
1418
1291
  default_port = 443 if scheme == "https" else 9200
1419
1292
  port = int(typer.prompt("Port", default=str(default_port)))
@@ -75,6 +75,8 @@ def serve(
75
75
  typer.echo(f" OpenSearch: {cfg.opensearch_host}:{cfg.opensearch_port}")
76
76
  typer.echo(f" Index: {cfg.index}")
77
77
 
78
+ from ..version import __version__
79
+ typer.echo(f" Version: {__version__}")
78
80
  typer.echo(f" Listening on: {host}:{port}")
79
81
  typer.echo()
80
82
 
@@ -5,6 +5,8 @@
5
5
  # - Ingest mode: write directly to OpenSearch
6
6
 
7
7
  import json
8
+ import platform
9
+ from contextlib import asynccontextmanager
8
10
  from typing import Optional
9
11
 
10
12
  from fastapi import FastAPI, Request, Response, HTTPException
@@ -17,6 +19,7 @@ from .schema import (
17
19
  validate_record,
18
20
  normalize_records,
19
21
  enrich_record,
22
+ get_current_timestamp,
20
23
  )
21
24
  from .errors import (
22
25
  CollectorError,
@@ -33,12 +36,73 @@ from .auth import (
33
36
  )
34
37
  from .forwarder import forward_request
35
38
  from .ingestor import ingest_records
39
+ from ..version import __version__
40
+
41
+ @asynccontextmanager
42
+ async def lifespan(app: FastAPI):
43
+ """Emit a startup trace to the index so operators can see when the collector started."""
44
+ cfg = load_config()
45
+ mode = cfg.get_collector_mode()
46
+
47
+ if mode == "ingest":
48
+ try:
49
+ client = get_opensearch_client()
50
+ doc = DevlogsRecord(
51
+ application="devlogs-collector",
52
+ component="lifecycle",
53
+ timestamp=get_current_timestamp(),
54
+ message="Collector started",
55
+ level="info",
56
+ area="startup",
57
+ version=__version__,
58
+ fields={
59
+ "mode": mode,
60
+ "host": platform.node(),
61
+ "opensearch_host": cfg.opensearch_host,
62
+ "index": cfg.index,
63
+ },
64
+ )
65
+ doc.collected_ts = get_current_timestamp()
66
+ doc.client_ip = "127.0.0.1"
67
+ doc._identity = {"mode": "internal"}
68
+ client.index(index=cfg.index, body=doc.to_dict())
69
+ except Exception:
70
+ # Startup trace is best-effort; don't block the server
71
+ pass
72
+
73
+ yield
74
+
75
+ # Emit shutdown trace
76
+ if mode == "ingest":
77
+ try:
78
+ client = get_opensearch_client()
79
+ doc = DevlogsRecord(
80
+ application="devlogs-collector",
81
+ component="lifecycle",
82
+ timestamp=get_current_timestamp(),
83
+ message="Collector stopped",
84
+ level="info",
85
+ area="shutdown",
86
+ version=__version__,
87
+ fields={
88
+ "mode": mode,
89
+ "host": platform.node(),
90
+ },
91
+ )
92
+ doc.collected_ts = get_current_timestamp()
93
+ doc.client_ip = "127.0.0.1"
94
+ doc._identity = {"mode": "internal"}
95
+ client.index(index=cfg.index, body=doc.to_dict())
96
+ except Exception:
97
+ pass
98
+
36
99
 
37
100
  # Create FastAPI app for collector
38
101
  app = FastAPI(
39
102
  title="Devlogs Collector",
40
103
  description="HTTP log collector for the devlogs format",
41
- version="1.0.0",
104
+ version=__version__,
105
+ lifespan=lifespan,
42
106
  )
43
107
 
44
108
 
@@ -83,6 +147,7 @@ async def health():
83
147
  return {
84
148
  "status": "healthy",
85
149
  "mode": mode,
150
+ "version": __version__,
86
151
  }
87
152
 
88
153
 
@@ -2,7 +2,11 @@
2
2
 
3
3
  import os
4
4
  import re
5
- from urllib.parse import urlparse, unquote
5
+ import sys
6
+ import warnings
7
+ from dataclasses import dataclass
8
+ from typing import Optional
9
+ from urllib.parse import urlparse, unquote, parse_qs
6
10
 
7
11
  # Lazy load dotenv - only when config is first accessed
8
12
  _dotenv_loaded = False
@@ -118,16 +122,180 @@ class URLParseError(ValueError):
118
122
  pass
119
123
 
120
124
 
125
+ @dataclass
126
+ class CollectorURLConfig:
127
+ """Parsed collector URL configuration."""
128
+ url: str # Base URL (scheme://host:port/path, no credentials)
129
+ token: Optional[str] = None # Bearer token
130
+
131
+
132
+ @dataclass
133
+ class OpenSearchURLConfig:
134
+ """Parsed OpenSearch URL configuration."""
135
+ scheme: str
136
+ host: str
137
+ port: int
138
+ user: Optional[str] = None
139
+ password: Optional[str] = None
140
+ index: Optional[str] = None
141
+ application: Optional[str] = None # Optional application filter from second path segment
142
+
143
+
144
+ def parse_url(url: str):
145
+ """Parse a URL and return a CollectorURLConfig or OpenSearchURLConfig.
146
+
147
+ Detection logic:
148
+ - opensearchs:// (TLS) or opensearch:// (non-TLS) → OpenSearch
149
+ - https:// or http:// with both user+pass → legacy OpenSearch (deprecation warning)
150
+ - https:// or http:// with token-only or ?token= → Collector
151
+
152
+ Returns: CollectorURLConfig or OpenSearchURLConfig
153
+ Raises: URLParseError if URL is malformed
154
+ """
155
+ if not url:
156
+ raise URLParseError("Empty URL")
157
+
158
+ # Check for opensearch:// or opensearchs:// schemes
159
+ if url.startswith("opensearchs://") or url.startswith("opensearch://"):
160
+ return _parse_opensearch_scheme_url(url)
161
+
162
+ parsed = urlparse(url)
163
+
164
+ if parsed.scheme not in ("http", "https"):
165
+ raise URLParseError(f"Invalid URL scheme '{parsed.scheme}': expected 'http', 'https', 'opensearch', or 'opensearchs'")
166
+
167
+ if not parsed.hostname:
168
+ raise URLParseError(f"Invalid URL '{url}': missing hostname")
169
+
170
+ # If both user AND password → legacy OpenSearch format, emit deprecation warning
171
+ if parsed.username and parsed.password:
172
+ warnings.warn(
173
+ f"Using https://user:pass@host URL format for OpenSearch is deprecated. "
174
+ f"Use opensearchs://user:pass@host instead.",
175
+ DeprecationWarning,
176
+ stacklevel=2,
177
+ )
178
+ return _parse_opensearch_url_to_config(url)
179
+
180
+ # Otherwise it's a collector URL
181
+ return _parse_collector_url_config(url)
182
+
183
+
184
+ def _parse_opensearch_scheme_url(url: str) -> OpenSearchURLConfig:
185
+ """Parse opensearchs:// (TLS) or opensearch:// (non-TLS) URL into OpenSearchURLConfig."""
186
+ # Determine the transport scheme
187
+ if url.startswith("opensearchs://"):
188
+ transport_scheme = "https"
189
+ parse_url_str = "https://" + url[len("opensearchs://"):]
190
+ else:
191
+ transport_scheme = "http"
192
+ parse_url_str = "http://" + url[len("opensearch://"):]
193
+
194
+ parsed = urlparse(parse_url_str)
195
+
196
+ if not parsed.hostname:
197
+ raise URLParseError(f"Invalid URL '{url}': missing hostname")
198
+
199
+ host = parsed.hostname
200
+ port = parsed.port or (443 if transport_scheme == "https" else 9200)
201
+ user = unquote(parsed.username) if parsed.username else None
202
+ password = unquote(parsed.password) if parsed.password else None
203
+
204
+ # Path segments: first = index, second = application
205
+ path = parsed.path.strip("/")
206
+ parts = path.split("/", 1) if path else []
207
+ index = parts[0] if parts else None
208
+ application = parts[1] if len(parts) > 1 else None
209
+
210
+ return OpenSearchURLConfig(
211
+ scheme=transport_scheme,
212
+ host=host,
213
+ port=port,
214
+ user=user,
215
+ password=password,
216
+ index=index or None,
217
+ application=application or None,
218
+ )
219
+
220
+
221
+ def _parse_collector_url_config(url: str) -> CollectorURLConfig:
222
+ """Parse an http(s) URL as a collector URL, extracting token."""
223
+ parsed = urlparse(url)
224
+
225
+ token = None
226
+
227
+ # Check for token in userinfo (token-only, no password)
228
+ if parsed.username and not parsed.password:
229
+ token = unquote(parsed.username)
230
+ # Rebuild URL without userinfo
231
+ if parsed.port:
232
+ netloc = f"{parsed.hostname}:{parsed.port}"
233
+ else:
234
+ netloc = parsed.hostname or ""
235
+ from urllib.parse import urlunparse
236
+ clean_url = urlunparse((
237
+ parsed.scheme, netloc, parsed.path,
238
+ parsed.params, parsed.query, parsed.fragment,
239
+ ))
240
+ else:
241
+ clean_url = url
242
+
243
+ # Check for ?token= query param
244
+ if not token:
245
+ query_params = parse_qs(parsed.query)
246
+ token_values = query_params.get("token")
247
+ if token_values:
248
+ token = token_values[0]
249
+ # Strip token param from the URL
250
+ from urllib.parse import urlencode
251
+ remaining = {k: v[0] for k, v in query_params.items() if k != "token"}
252
+ new_query = urlencode(remaining) if remaining else ""
253
+ reparsed = urlparse(clean_url)
254
+ from urllib.parse import urlunparse
255
+ clean_url = urlunparse((
256
+ reparsed.scheme, reparsed.netloc, reparsed.path,
257
+ reparsed.params, new_query, reparsed.fragment,
258
+ ))
259
+
260
+ return CollectorURLConfig(url=clean_url, token=token)
261
+
262
+
263
+ def _parse_opensearch_url_to_config(url: str) -> OpenSearchURLConfig:
264
+ """Parse a standard http(s) URL into OpenSearchURLConfig."""
265
+ result = _parse_opensearch_url(url)
266
+ if result is None:
267
+ raise URLParseError(f"Invalid URL '{url}'")
268
+ scheme, host, port, user, password, index = result
269
+ return OpenSearchURLConfig(
270
+ scheme=scheme, host=host, port=port,
271
+ user=user, password=password, index=index,
272
+ )
273
+
274
+
121
275
  def _parse_opensearch_url(url: str):
122
276
  """Parse DEVLOGS_OPENSEARCH_URL into components.
123
277
 
124
278
  Supports format: https://user:pass@host:port/index
279
+ Also supports: opensearchs://user:pass@host:port/index (TLS)
280
+ opensearch://user:pass@host:port/index (non-TLS)
125
281
  Returns: (scheme, host, port, user, pass, index) or None if no URL
126
282
  Raises: URLParseError if URL is malformed
127
283
  """
128
284
  if not url:
129
285
  return None
130
- parsed = urlparse(url)
286
+
287
+ # Handle opensearchs:// (TLS) and opensearch:// (non-TLS) schemes
288
+ if url.startswith("opensearchs://"):
289
+ actual_url = "https://" + url[len("opensearchs://"):]
290
+ force_scheme = "https"
291
+ elif url.startswith("opensearch://"):
292
+ actual_url = "http://" + url[len("opensearch://"):]
293
+ force_scheme = "http"
294
+ else:
295
+ actual_url = url
296
+ force_scheme = None
297
+
298
+ parsed = urlparse(actual_url)
131
299
 
132
300
  # Validate scheme
133
301
  if parsed.scheme and parsed.scheme not in ("http", "https"):
@@ -137,7 +305,7 @@ def _parse_opensearch_url(url: str):
137
305
  if not parsed.hostname:
138
306
  raise URLParseError(f"Invalid URL '{url}': missing hostname")
139
307
 
140
- scheme = parsed.scheme or "http"
308
+ scheme = force_scheme or parsed.scheme or "http"
141
309
  host = parsed.hostname
142
310
  port = parsed.port or (443 if scheme == "https" else 9200)
143
311
  # URL-decode username and password since urlparse doesn't do this automatically
@@ -227,6 +395,15 @@ class DevlogsConfig:
227
395
  self.collector_workers = int(_getenv("DEVLOGS_COLLECTOR_WORKERS", "1"))
228
396
  self.collector_log_level = _getenv("DEVLOGS_COLLECTOR_LOG_LEVEL", "info")
229
397
 
398
+ @property
399
+ def url_mode(self) -> str:
400
+ """Return 'collector' if DEVLOGS_URL is set, 'opensearch' if OpenSearch is configured, else 'none'."""
401
+ if self.collector_url:
402
+ return "collector"
403
+ if self.has_opensearch_config():
404
+ return "opensearch"
405
+ return "none"
406
+
230
407
  def has_opensearch_config(self) -> bool:
231
408
  """Check if OpenSearch admin connection is configured."""
232
409
  return bool(
@@ -257,8 +434,27 @@ def set_dotenv_path(path: str):
257
434
 
258
435
 
259
436
  def set_url(url: str):
260
- """Set the OpenSearch URL. Must be called before load_config()."""
261
- os.environ["DEVLOGS_OPENSEARCH_URL"] = url
437
+ """Set the URL, auto-detecting whether it's a collector or OpenSearch URL.
438
+
439
+ Uses parse_url() to detect the URL type:
440
+ - Collector URLs → sets DEVLOGS_URL
441
+ - OpenSearch URLs → sets DEVLOGS_OPENSEARCH_URL
442
+ """
443
+ if not url:
444
+ return
445
+ try:
446
+ parsed = parse_url(url)
447
+ except URLParseError:
448
+ # Fall back to legacy behavior
449
+ os.environ["DEVLOGS_OPENSEARCH_URL"] = url
450
+ return
451
+
452
+ if isinstance(parsed, CollectorURLConfig):
453
+ # Reconstruct the full collector URL for DEVLOGS_URL
454
+ os.environ["DEVLOGS_URL"] = url
455
+ else:
456
+ # OpenSearch URL - reconstruct with the correct scheme for _parse_opensearch_url
457
+ os.environ["DEVLOGS_OPENSEARCH_URL"] = url
262
458
 
263
459
  def load_config() -> DevlogsConfig:
264
460
  """Return a config object with all settings loaded."""
@@ -13,29 +13,42 @@ from .handler import OpenSearchHandler
13
13
  def run_demo(
14
14
  duration: int,
15
15
  count: int,
16
- require_opensearch,
16
+ require_opensearch=None,
17
+ collector_url: str = None,
17
18
  ):
18
19
  """Generate demo logs to illustrate devlogs capabilities."""
19
20
  cfg = load_config()
20
21
 
21
- # Show loaded configuration
22
- typer.echo("=== DevLogs Demo ===\n")
23
- typer.echo("Configuration loaded from .env:")
24
- typer.echo(f" DEVLOGS_OPENSEARCH_HOST: {cfg.opensearch_host}")
25
- typer.echo(f" DEVLOGS_OPENSEARCH_PORT: {cfg.opensearch_port}")
26
- typer.echo(f" DEVLOGS_OPENSEARCH_USER: {cfg.opensearch_user}")
27
- typer.echo(f" DEVLOGS_OPENSEARCH_PASS: {'*' * len(cfg.opensearch_pass)}")
28
- typer.echo(f" DEVLOGS_INDEX: {cfg.index}")
29
- typer.echo(f" DEVLOGS_RETENTION_DEBUG: {cfg.retention_debug_hours}h")
30
- typer.echo("")
31
-
32
- # Check OpenSearch connection and index
33
- client, cfg = require_opensearch()
34
- handler = OpenSearchHandler(
35
- level=logging.DEBUG,
36
- opensearch_client=client,
37
- index_name=cfg.index,
38
- )
22
+ if collector_url:
23
+ # Collector mode
24
+ typer.echo("=== DevLogs Demo (Collector Mode) ===\n")
25
+ typer.echo(f" Collector URL: {collector_url}")
26
+ typer.echo("")
27
+
28
+ handler = OpenSearchHandler(
29
+ level=logging.DEBUG,
30
+ collector_url=collector_url,
31
+ )
32
+ else:
33
+ # OpenSearch mode
34
+ typer.echo("=== DevLogs Demo ===\n")
35
+ typer.echo("Configuration loaded from .env:")
36
+ typer.echo(f" DEVLOGS_OPENSEARCH_HOST: {cfg.opensearch_host}")
37
+ typer.echo(f" DEVLOGS_OPENSEARCH_PORT: {cfg.opensearch_port}")
38
+ typer.echo(f" DEVLOGS_OPENSEARCH_USER: {cfg.opensearch_user}")
39
+ typer.echo(f" DEVLOGS_OPENSEARCH_PASS: {'*' * len(cfg.opensearch_pass)}")
40
+ typer.echo(f" DEVLOGS_INDEX: {cfg.index}")
41
+ typer.echo(f" DEVLOGS_RETENTION_DEBUG: {cfg.retention_debug_hours}h")
42
+ typer.echo("")
43
+
44
+ # Check OpenSearch connection and index
45
+ client, cfg = require_opensearch()
46
+ handler = OpenSearchHandler(
47
+ level=logging.DEBUG,
48
+ opensearch_client=client,
49
+ index_name=cfg.index,
50
+ )
51
+
39
52
  handler.setFormatter(logging.Formatter("%(message)s"))
40
53
 
41
54
  logger = logging.getLogger("devlogs.demo")