devlogs 2.0.2__tar.gz → 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {devlogs-2.0.2/src/devlogs.egg-info → devlogs-2.1.0}/PKG-INFO +1 -1
- {devlogs-2.0.2 → devlogs-2.1.0}/pyproject.toml +1 -1
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/cli.py +32 -159
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/collector/cli.py +2 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/collector/server.py +66 -1
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/config.py +201 -5
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/demo.py +32 -19
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/devlogs_client.py +21 -5
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/handler.py +44 -4
- devlogs-2.1.0/src/devlogs/jenkins/cli.py +102 -0
- devlogs-2.1.0/src/devlogs/version.py +3 -0
- {devlogs-2.0.2 → devlogs-2.1.0/src/devlogs.egg-info}/PKG-INFO +1 -1
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs.egg-info/SOURCES.txt +2 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_cli.py +16 -26
- {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_config.py +8 -26
- {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_mcp_server.py +22 -21
- devlogs-2.1.0/tests/test_url_parsing.py +239 -0
- devlogs-2.0.2/src/devlogs/jenkins/cli.py +0 -257
- {devlogs-2.0.2 → devlogs-2.1.0}/LICENSE +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/MANIFEST.in +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/README.md +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/setup.cfg +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/__init__.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/__main__.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/build_info.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/collector/__init__.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/collector/auth.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/collector/errors.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/collector/forwarder.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/collector/ingestor.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/collector/schema.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/context.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/formatting.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/jenkins/__init__.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/jenkins/core.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/levels.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/mcp/__init__.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/mcp/server.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/opensearch/__init__.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/opensearch/client.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/opensearch/indexing.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/opensearch/mappings.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/opensearch/queries.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/retention.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/scrub.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/time_utils.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/web/__init__.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/web/server.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/web/static/devlogs.css +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/web/static/devlogs.js +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/web/static/index.html +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs/wrapper.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs.egg-info/dependency_links.txt +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs.egg-info/entry_points.txt +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs.egg-info/requires.txt +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/src/devlogs.egg-info/top_level.txt +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_build_info.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_collector_auth.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_collector_config.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_collector_schema.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_collector_server.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_context.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_devlogs_client.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_formatting.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_handler.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_indexing.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_levels.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_mappings.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_opensearch_client.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_opensearch_queries.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_retention.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_scrub.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_time_utils.py +0 -0
- {devlogs-2.0.2 → devlogs-2.1.0}/tests/test_web.py +0 -0
|
@@ -10,7 +10,7 @@ import click
|
|
|
10
10
|
import typer
|
|
11
11
|
from pathlib import Path
|
|
12
12
|
|
|
13
|
-
from .config import load_config, set_dotenv_path, set_url, URLParseError, _parse_opensearch_url
|
|
13
|
+
from .config import load_config, set_dotenv_path, set_url, URLParseError, _parse_opensearch_url, parse_url, CollectorURLConfig
|
|
14
14
|
from .formatting import format_timestamp
|
|
15
15
|
from .opensearch.client import (
|
|
16
16
|
get_opensearch_client,
|
|
@@ -39,7 +39,7 @@ OLD_TEMPLATE_NAMES = ("devlogs-template", "devlogs-logs-template")
|
|
|
39
39
|
|
|
40
40
|
# Common options for commands - these can be placed anywhere in the command line
|
|
41
41
|
ENV_OPTION = typer.Option(None, "--env", help="Path to .env file to load")
|
|
42
|
-
URL_OPTION = typer.Option(None, "--url", help="
|
|
42
|
+
URL_OPTION = typer.Option(None, "--url", help="URL (e.g., opensearchs://user:pass@host:port/index or https://TOKEN@host:port/path)")
|
|
43
43
|
|
|
44
44
|
|
|
45
45
|
def _apply_common_options(env: str = None, url: str = None):
|
|
@@ -61,11 +61,17 @@ def main_callback(
|
|
|
61
61
|
_apply_common_options(env, url)
|
|
62
62
|
|
|
63
63
|
|
|
64
|
+
_HIDDEN_FIELDS = {"run_id", "build_number", "build_url", "job", "node_id", "seq"}
|
|
65
|
+
|
|
66
|
+
|
|
64
67
|
def _format_features(features):
|
|
65
68
|
if not features:
|
|
66
69
|
return ""
|
|
67
70
|
if isinstance(features, dict):
|
|
68
|
-
items = sorted(
|
|
71
|
+
items = sorted(
|
|
72
|
+
((k, v) for k, v in features.items() if k not in _HIDDEN_FIELDS),
|
|
73
|
+
key=lambda item: str(item[0]),
|
|
74
|
+
)
|
|
69
75
|
parts = []
|
|
70
76
|
for key, value in items:
|
|
71
77
|
key_text = str(key)
|
|
@@ -1166,7 +1172,13 @@ def demo(
|
|
|
1166
1172
|
"""Generate demo logs to illustrate devlogs capabilities."""
|
|
1167
1173
|
_apply_common_options(env, url)
|
|
1168
1174
|
from .demo import run_demo
|
|
1169
|
-
|
|
1175
|
+
|
|
1176
|
+
cfg = load_config()
|
|
1177
|
+
if cfg.url_mode == "collector":
|
|
1178
|
+
from .devlogs_client import DevlogsClient
|
|
1179
|
+
run_demo(duration, count, require_opensearch=None, collector_url=cfg.collector_url)
|
|
1180
|
+
else:
|
|
1181
|
+
run_demo(duration, count, require_opensearch=require_opensearch)
|
|
1170
1182
|
|
|
1171
1183
|
|
|
1172
1184
|
@app.command()
|
|
@@ -1181,8 +1193,20 @@ def serve(
|
|
|
1181
1193
|
|
|
1182
1194
|
|
|
1183
1195
|
def _build_opensearch_url(scheme: str, host: str, port: int, user: str, password: str, index: str) -> str:
|
|
1184
|
-
"""Build an OpenSearch URL from components, URL-encoding credentials.
|
|
1196
|
+
"""Build an OpenSearch URL from components, URL-encoding credentials.
|
|
1197
|
+
|
|
1198
|
+
Uses the opensearchs:// scheme (TLS) or opensearch:// (non-TLS).
|
|
1199
|
+
The scheme parameter accepts 'https'/'http' (mapped to opensearch schemes)
|
|
1200
|
+
or 'opensearchs'/'opensearch' directly.
|
|
1201
|
+
"""
|
|
1185
1202
|
from urllib.parse import quote
|
|
1203
|
+
# Map legacy schemes to opensearch:// variants
|
|
1204
|
+
if scheme == "https":
|
|
1205
|
+
url_scheme = "opensearchs"
|
|
1206
|
+
elif scheme == "http":
|
|
1207
|
+
url_scheme = "opensearch"
|
|
1208
|
+
else:
|
|
1209
|
+
url_scheme = scheme
|
|
1186
1210
|
# URL-encode username and password to handle special characters
|
|
1187
1211
|
encoded_user = quote(user, safe="") if user else ""
|
|
1188
1212
|
encoded_pass = quote(password, safe="") if password else ""
|
|
@@ -1193,7 +1217,7 @@ def _build_opensearch_url(scheme: str, host: str, port: int, user: str, password
|
|
|
1193
1217
|
else:
|
|
1194
1218
|
auth = ""
|
|
1195
1219
|
path = f"/{index}" if index else ""
|
|
1196
|
-
return f"{
|
|
1220
|
+
return f"{url_scheme}://{auth}{host}:{port}{path}"
|
|
1197
1221
|
|
|
1198
1222
|
|
|
1199
1223
|
def _format_env_output(scheme: str, host: str, port: int, user: str, password: str, index: str) -> str:
|
|
@@ -1211,155 +1235,6 @@ def _format_env_output(scheme: str, host: str, port: int, user: str, password: s
|
|
|
1211
1235
|
return "\n".join(lines)
|
|
1212
1236
|
|
|
1213
1237
|
|
|
1214
|
-
@app.command()
|
|
1215
|
-
def initjenkins(
|
|
1216
|
-
jenkinsfile: str = typer.Argument("Jenkinsfile", help="Path to Jenkinsfile to modify"),
|
|
1217
|
-
credential_id: str = typer.Option("devlogs-opensearch-url", "--credential-id", "-c", help="Jenkins credential ID to use"),
|
|
1218
|
-
env: str = ENV_OPTION,
|
|
1219
|
-
url: str = URL_OPTION,
|
|
1220
|
-
):
|
|
1221
|
-
"""Add devlogs configuration to an existing Jenkinsfile.
|
|
1222
|
-
|
|
1223
|
-
This command modifies an existing Jenkinsfile to add an options block
|
|
1224
|
-
with the devlogs pipeline step configured to use a Jenkins credential.
|
|
1225
|
-
|
|
1226
|
-
After running this command, you need to create a Jenkins credential
|
|
1227
|
-
of type "Secret text" with the OpenSearch URL.
|
|
1228
|
-
|
|
1229
|
-
Examples:
|
|
1230
|
-
devlogs initjenkins # Modify ./Jenkinsfile
|
|
1231
|
-
devlogs initjenkins path/to/Jenkinsfile # Modify specific file
|
|
1232
|
-
devlogs initjenkins --credential-id my-cred # Use custom credential ID
|
|
1233
|
-
"""
|
|
1234
|
-
import re
|
|
1235
|
-
|
|
1236
|
-
_apply_common_options(env, url)
|
|
1237
|
-
|
|
1238
|
-
jenkinsfile_path = Path(jenkinsfile)
|
|
1239
|
-
if not jenkinsfile_path.is_file():
|
|
1240
|
-
typer.echo(typer.style(f"Error: Jenkinsfile not found: {jenkinsfile_path}", fg=typer.colors.RED), err=True)
|
|
1241
|
-
raise typer.Exit(1)
|
|
1242
|
-
|
|
1243
|
-
content = jenkinsfile_path.read_text(encoding="utf-8")
|
|
1244
|
-
|
|
1245
|
-
# Check if this looks like a declarative pipeline
|
|
1246
|
-
if "pipeline" not in content:
|
|
1247
|
-
typer.echo(typer.style("Error: File does not appear to be a declarative Jenkins pipeline.", fg=typer.colors.RED), err=True)
|
|
1248
|
-
typer.echo("This command only supports declarative pipelines with a 'pipeline { }' block.", err=True)
|
|
1249
|
-
raise typer.Exit(1)
|
|
1250
|
-
|
|
1251
|
-
# Check if devlogs is already configured
|
|
1252
|
-
if "devlogs(" in content:
|
|
1253
|
-
typer.echo(typer.style("Warning: Jenkinsfile already appears to have devlogs configuration.", fg=typer.colors.YELLOW))
|
|
1254
|
-
typer.echo("Review the file manually to ensure correct configuration.")
|
|
1255
|
-
raise typer.Exit(0)
|
|
1256
|
-
|
|
1257
|
-
options_line = f" devlogs(credentialsId: '{credential_id}')"
|
|
1258
|
-
|
|
1259
|
-
modified = False
|
|
1260
|
-
lines = content.split("\n")
|
|
1261
|
-
result_lines = []
|
|
1262
|
-
i = 0
|
|
1263
|
-
|
|
1264
|
-
# Track brace depth and whether we're inside pipeline block
|
|
1265
|
-
in_pipeline = False
|
|
1266
|
-
pipeline_brace_depth = 0
|
|
1267
|
-
added_options = False
|
|
1268
|
-
|
|
1269
|
-
while i < len(lines):
|
|
1270
|
-
line = lines[i]
|
|
1271
|
-
result_lines.append(line)
|
|
1272
|
-
|
|
1273
|
-
# Detect entering pipeline block
|
|
1274
|
-
if not in_pipeline and re.match(r'^\s*pipeline\s*\{', line):
|
|
1275
|
-
in_pipeline = True
|
|
1276
|
-
pipeline_brace_depth = 1
|
|
1277
|
-
i += 1
|
|
1278
|
-
continue
|
|
1279
|
-
|
|
1280
|
-
if in_pipeline:
|
|
1281
|
-
# Count braces to track depth
|
|
1282
|
-
pipeline_brace_depth += line.count('{') - line.count('}')
|
|
1283
|
-
|
|
1284
|
-
# Check for existing options block and add our line
|
|
1285
|
-
if not added_options and re.match(r'^\s*options\s*\{', line):
|
|
1286
|
-
result_lines.append(options_line)
|
|
1287
|
-
added_options = True
|
|
1288
|
-
modified = True
|
|
1289
|
-
|
|
1290
|
-
# If we hit stages and haven't added options, add it before stages
|
|
1291
|
-
if re.match(r'^\s*stages\s*\{', line) and not added_options:
|
|
1292
|
-
# Insert before the stages line
|
|
1293
|
-
result_lines.pop() # Remove the stages line we just added
|
|
1294
|
-
|
|
1295
|
-
result_lines.append("")
|
|
1296
|
-
result_lines.append(" options {")
|
|
1297
|
-
result_lines.append(options_line)
|
|
1298
|
-
result_lines.append(" }")
|
|
1299
|
-
added_options = True
|
|
1300
|
-
modified = True
|
|
1301
|
-
|
|
1302
|
-
result_lines.append("")
|
|
1303
|
-
result_lines.append(line) # Re-add the stages line
|
|
1304
|
-
|
|
1305
|
-
# Exit pipeline tracking when we close the pipeline block
|
|
1306
|
-
if pipeline_brace_depth == 0:
|
|
1307
|
-
in_pipeline = False
|
|
1308
|
-
|
|
1309
|
-
i += 1
|
|
1310
|
-
|
|
1311
|
-
if not modified:
|
|
1312
|
-
typer.echo(typer.style("Error: Could not find a suitable location to add devlogs configuration.", fg=typer.colors.RED), err=True)
|
|
1313
|
-
typer.echo("Ensure the Jenkinsfile has a 'pipeline { stages { } }' structure.", err=True)
|
|
1314
|
-
raise typer.Exit(1)
|
|
1315
|
-
|
|
1316
|
-
# Write the modified file
|
|
1317
|
-
new_content = "\n".join(result_lines)
|
|
1318
|
-
jenkinsfile_path.write_text(new_content, encoding="utf-8")
|
|
1319
|
-
|
|
1320
|
-
typer.echo(typer.style(f"Modified {jenkinsfile_path}", fg=typer.colors.GREEN))
|
|
1321
|
-
typer.echo()
|
|
1322
|
-
typer.echo("Added:")
|
|
1323
|
-
typer.echo(f" - Options: devlogs(credentialsId: '{credential_id}')")
|
|
1324
|
-
|
|
1325
|
-
# Print setup instructions
|
|
1326
|
-
typer.echo()
|
|
1327
|
-
typer.echo(typer.style("Next steps - Create Jenkins credential:", fg=typer.colors.CYAN, bold=True))
|
|
1328
|
-
typer.echo("=" * 60)
|
|
1329
|
-
typer.echo()
|
|
1330
|
-
typer.echo("1. Go to Jenkins > Manage Jenkins > Credentials")
|
|
1331
|
-
typer.echo("2. Select the appropriate domain (e.g., Global)")
|
|
1332
|
-
typer.echo("3. Click 'Add Credentials'")
|
|
1333
|
-
typer.echo("4. Configure:")
|
|
1334
|
-
typer.echo(f" - Kind: Secret text")
|
|
1335
|
-
typer.echo(f" - ID: {credential_id}")
|
|
1336
|
-
typer.echo(" - Secret: <your OpenSearch URL>")
|
|
1337
|
-
typer.echo()
|
|
1338
|
-
|
|
1339
|
-
# Try to load config and show the URL value
|
|
1340
|
-
try:
|
|
1341
|
-
cfg = load_config()
|
|
1342
|
-
if cfg.enabled:
|
|
1343
|
-
# Build the URL from config
|
|
1344
|
-
credential_url = _build_opensearch_url(
|
|
1345
|
-
cfg.opensearch_scheme,
|
|
1346
|
-
cfg.opensearch_host,
|
|
1347
|
-
cfg.opensearch_port,
|
|
1348
|
-
cfg.opensearch_user,
|
|
1349
|
-
cfg.opensearch_pass,
|
|
1350
|
-
cfg.index,
|
|
1351
|
-
)
|
|
1352
|
-
typer.echo(typer.style("Credential value (from your .env/environment):", fg=typer.colors.GREEN, bold=True))
|
|
1353
|
-
typer.echo("-" * 60)
|
|
1354
|
-
typer.echo(credential_url)
|
|
1355
|
-
typer.echo("-" * 60)
|
|
1356
|
-
else:
|
|
1357
|
-
typer.echo("Tip: Set up a .env file with DEVLOGS_OPENSEARCH_URL to see the exact value here.")
|
|
1358
|
-
typer.echo(" Run 'devlogs mkurl' to interactively build the URL.")
|
|
1359
|
-
except Exception:
|
|
1360
|
-
typer.echo("Tip: Run 'devlogs mkurl' to interactively build the OpenSearch URL.")
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
1238
|
@app.command()
|
|
1364
1239
|
def mkurl():
|
|
1365
1240
|
"""Interactively create an OpenSearch URL and show .env formats.
|
|
@@ -1410,10 +1285,8 @@ def mkurl():
|
|
|
1410
1285
|
else:
|
|
1411
1286
|
# Prompt for each component
|
|
1412
1287
|
typer.echo()
|
|
1413
|
-
|
|
1414
|
-
if
|
|
1415
|
-
typer.echo(typer.style("Error: Scheme must be 'http' or 'https'.", fg=typer.colors.RED), err=True)
|
|
1416
|
-
raise typer.Exit(1)
|
|
1288
|
+
use_tls = typer.prompt("Use TLS? (yes/no)", default="yes")
|
|
1289
|
+
scheme = "https" if use_tls.lower() in ("yes", "y") else "http"
|
|
1417
1290
|
host = typer.prompt("Host", default="localhost")
|
|
1418
1291
|
default_port = 443 if scheme == "https" else 9200
|
|
1419
1292
|
port = int(typer.prompt("Port", default=str(default_port)))
|
|
@@ -75,6 +75,8 @@ def serve(
|
|
|
75
75
|
typer.echo(f" OpenSearch: {cfg.opensearch_host}:{cfg.opensearch_port}")
|
|
76
76
|
typer.echo(f" Index: {cfg.index}")
|
|
77
77
|
|
|
78
|
+
from ..version import __version__
|
|
79
|
+
typer.echo(f" Version: {__version__}")
|
|
78
80
|
typer.echo(f" Listening on: {host}:{port}")
|
|
79
81
|
typer.echo()
|
|
80
82
|
|
|
@@ -5,6 +5,8 @@
|
|
|
5
5
|
# - Ingest mode: write directly to OpenSearch
|
|
6
6
|
|
|
7
7
|
import json
|
|
8
|
+
import platform
|
|
9
|
+
from contextlib import asynccontextmanager
|
|
8
10
|
from typing import Optional
|
|
9
11
|
|
|
10
12
|
from fastapi import FastAPI, Request, Response, HTTPException
|
|
@@ -17,6 +19,7 @@ from .schema import (
|
|
|
17
19
|
validate_record,
|
|
18
20
|
normalize_records,
|
|
19
21
|
enrich_record,
|
|
22
|
+
get_current_timestamp,
|
|
20
23
|
)
|
|
21
24
|
from .errors import (
|
|
22
25
|
CollectorError,
|
|
@@ -33,12 +36,73 @@ from .auth import (
|
|
|
33
36
|
)
|
|
34
37
|
from .forwarder import forward_request
|
|
35
38
|
from .ingestor import ingest_records
|
|
39
|
+
from ..version import __version__
|
|
40
|
+
|
|
41
|
+
@asynccontextmanager
|
|
42
|
+
async def lifespan(app: FastAPI):
|
|
43
|
+
"""Emit a startup trace to the index so operators can see when the collector started."""
|
|
44
|
+
cfg = load_config()
|
|
45
|
+
mode = cfg.get_collector_mode()
|
|
46
|
+
|
|
47
|
+
if mode == "ingest":
|
|
48
|
+
try:
|
|
49
|
+
client = get_opensearch_client()
|
|
50
|
+
doc = DevlogsRecord(
|
|
51
|
+
application="devlogs-collector",
|
|
52
|
+
component="lifecycle",
|
|
53
|
+
timestamp=get_current_timestamp(),
|
|
54
|
+
message="Collector started",
|
|
55
|
+
level="info",
|
|
56
|
+
area="startup",
|
|
57
|
+
version=__version__,
|
|
58
|
+
fields={
|
|
59
|
+
"mode": mode,
|
|
60
|
+
"host": platform.node(),
|
|
61
|
+
"opensearch_host": cfg.opensearch_host,
|
|
62
|
+
"index": cfg.index,
|
|
63
|
+
},
|
|
64
|
+
)
|
|
65
|
+
doc.collected_ts = get_current_timestamp()
|
|
66
|
+
doc.client_ip = "127.0.0.1"
|
|
67
|
+
doc._identity = {"mode": "internal"}
|
|
68
|
+
client.index(index=cfg.index, body=doc.to_dict())
|
|
69
|
+
except Exception:
|
|
70
|
+
# Startup trace is best-effort; don't block the server
|
|
71
|
+
pass
|
|
72
|
+
|
|
73
|
+
yield
|
|
74
|
+
|
|
75
|
+
# Emit shutdown trace
|
|
76
|
+
if mode == "ingest":
|
|
77
|
+
try:
|
|
78
|
+
client = get_opensearch_client()
|
|
79
|
+
doc = DevlogsRecord(
|
|
80
|
+
application="devlogs-collector",
|
|
81
|
+
component="lifecycle",
|
|
82
|
+
timestamp=get_current_timestamp(),
|
|
83
|
+
message="Collector stopped",
|
|
84
|
+
level="info",
|
|
85
|
+
area="shutdown",
|
|
86
|
+
version=__version__,
|
|
87
|
+
fields={
|
|
88
|
+
"mode": mode,
|
|
89
|
+
"host": platform.node(),
|
|
90
|
+
},
|
|
91
|
+
)
|
|
92
|
+
doc.collected_ts = get_current_timestamp()
|
|
93
|
+
doc.client_ip = "127.0.0.1"
|
|
94
|
+
doc._identity = {"mode": "internal"}
|
|
95
|
+
client.index(index=cfg.index, body=doc.to_dict())
|
|
96
|
+
except Exception:
|
|
97
|
+
pass
|
|
98
|
+
|
|
36
99
|
|
|
37
100
|
# Create FastAPI app for collector
|
|
38
101
|
app = FastAPI(
|
|
39
102
|
title="Devlogs Collector",
|
|
40
103
|
description="HTTP log collector for the devlogs format",
|
|
41
|
-
version=
|
|
104
|
+
version=__version__,
|
|
105
|
+
lifespan=lifespan,
|
|
42
106
|
)
|
|
43
107
|
|
|
44
108
|
|
|
@@ -83,6 +147,7 @@ async def health():
|
|
|
83
147
|
return {
|
|
84
148
|
"status": "healthy",
|
|
85
149
|
"mode": mode,
|
|
150
|
+
"version": __version__,
|
|
86
151
|
}
|
|
87
152
|
|
|
88
153
|
|
|
@@ -2,7 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
import os
|
|
4
4
|
import re
|
|
5
|
-
|
|
5
|
+
import sys
|
|
6
|
+
import warnings
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from typing import Optional
|
|
9
|
+
from urllib.parse import urlparse, unquote, parse_qs
|
|
6
10
|
|
|
7
11
|
# Lazy load dotenv - only when config is first accessed
|
|
8
12
|
_dotenv_loaded = False
|
|
@@ -118,16 +122,180 @@ class URLParseError(ValueError):
|
|
|
118
122
|
pass
|
|
119
123
|
|
|
120
124
|
|
|
125
|
+
@dataclass
|
|
126
|
+
class CollectorURLConfig:
|
|
127
|
+
"""Parsed collector URL configuration."""
|
|
128
|
+
url: str # Base URL (scheme://host:port/path, no credentials)
|
|
129
|
+
token: Optional[str] = None # Bearer token
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
@dataclass
|
|
133
|
+
class OpenSearchURLConfig:
|
|
134
|
+
"""Parsed OpenSearch URL configuration."""
|
|
135
|
+
scheme: str
|
|
136
|
+
host: str
|
|
137
|
+
port: int
|
|
138
|
+
user: Optional[str] = None
|
|
139
|
+
password: Optional[str] = None
|
|
140
|
+
index: Optional[str] = None
|
|
141
|
+
application: Optional[str] = None # Optional application filter from second path segment
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def parse_url(url: str):
|
|
145
|
+
"""Parse a URL and return a CollectorURLConfig or OpenSearchURLConfig.
|
|
146
|
+
|
|
147
|
+
Detection logic:
|
|
148
|
+
- opensearchs:// (TLS) or opensearch:// (non-TLS) → OpenSearch
|
|
149
|
+
- https:// or http:// with both user+pass → legacy OpenSearch (deprecation warning)
|
|
150
|
+
- https:// or http:// with token-only or ?token= → Collector
|
|
151
|
+
|
|
152
|
+
Returns: CollectorURLConfig or OpenSearchURLConfig
|
|
153
|
+
Raises: URLParseError if URL is malformed
|
|
154
|
+
"""
|
|
155
|
+
if not url:
|
|
156
|
+
raise URLParseError("Empty URL")
|
|
157
|
+
|
|
158
|
+
# Check for opensearch:// or opensearchs:// schemes
|
|
159
|
+
if url.startswith("opensearchs://") or url.startswith("opensearch://"):
|
|
160
|
+
return _parse_opensearch_scheme_url(url)
|
|
161
|
+
|
|
162
|
+
parsed = urlparse(url)
|
|
163
|
+
|
|
164
|
+
if parsed.scheme not in ("http", "https"):
|
|
165
|
+
raise URLParseError(f"Invalid URL scheme '{parsed.scheme}': expected 'http', 'https', 'opensearch', or 'opensearchs'")
|
|
166
|
+
|
|
167
|
+
if not parsed.hostname:
|
|
168
|
+
raise URLParseError(f"Invalid URL '{url}': missing hostname")
|
|
169
|
+
|
|
170
|
+
# If both user AND password → legacy OpenSearch format, emit deprecation warning
|
|
171
|
+
if parsed.username and parsed.password:
|
|
172
|
+
warnings.warn(
|
|
173
|
+
f"Using https://user:pass@host URL format for OpenSearch is deprecated. "
|
|
174
|
+
f"Use opensearchs://user:pass@host instead.",
|
|
175
|
+
DeprecationWarning,
|
|
176
|
+
stacklevel=2,
|
|
177
|
+
)
|
|
178
|
+
return _parse_opensearch_url_to_config(url)
|
|
179
|
+
|
|
180
|
+
# Otherwise it's a collector URL
|
|
181
|
+
return _parse_collector_url_config(url)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _parse_opensearch_scheme_url(url: str) -> OpenSearchURLConfig:
|
|
185
|
+
"""Parse opensearchs:// (TLS) or opensearch:// (non-TLS) URL into OpenSearchURLConfig."""
|
|
186
|
+
# Determine the transport scheme
|
|
187
|
+
if url.startswith("opensearchs://"):
|
|
188
|
+
transport_scheme = "https"
|
|
189
|
+
parse_url_str = "https://" + url[len("opensearchs://"):]
|
|
190
|
+
else:
|
|
191
|
+
transport_scheme = "http"
|
|
192
|
+
parse_url_str = "http://" + url[len("opensearch://"):]
|
|
193
|
+
|
|
194
|
+
parsed = urlparse(parse_url_str)
|
|
195
|
+
|
|
196
|
+
if not parsed.hostname:
|
|
197
|
+
raise URLParseError(f"Invalid URL '{url}': missing hostname")
|
|
198
|
+
|
|
199
|
+
host = parsed.hostname
|
|
200
|
+
port = parsed.port or (443 if transport_scheme == "https" else 9200)
|
|
201
|
+
user = unquote(parsed.username) if parsed.username else None
|
|
202
|
+
password = unquote(parsed.password) if parsed.password else None
|
|
203
|
+
|
|
204
|
+
# Path segments: first = index, second = application
|
|
205
|
+
path = parsed.path.strip("/")
|
|
206
|
+
parts = path.split("/", 1) if path else []
|
|
207
|
+
index = parts[0] if parts else None
|
|
208
|
+
application = parts[1] if len(parts) > 1 else None
|
|
209
|
+
|
|
210
|
+
return OpenSearchURLConfig(
|
|
211
|
+
scheme=transport_scheme,
|
|
212
|
+
host=host,
|
|
213
|
+
port=port,
|
|
214
|
+
user=user,
|
|
215
|
+
password=password,
|
|
216
|
+
index=index or None,
|
|
217
|
+
application=application or None,
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _parse_collector_url_config(url: str) -> CollectorURLConfig:
|
|
222
|
+
"""Parse an http(s) URL as a collector URL, extracting token."""
|
|
223
|
+
parsed = urlparse(url)
|
|
224
|
+
|
|
225
|
+
token = None
|
|
226
|
+
|
|
227
|
+
# Check for token in userinfo (token-only, no password)
|
|
228
|
+
if parsed.username and not parsed.password:
|
|
229
|
+
token = unquote(parsed.username)
|
|
230
|
+
# Rebuild URL without userinfo
|
|
231
|
+
if parsed.port:
|
|
232
|
+
netloc = f"{parsed.hostname}:{parsed.port}"
|
|
233
|
+
else:
|
|
234
|
+
netloc = parsed.hostname or ""
|
|
235
|
+
from urllib.parse import urlunparse
|
|
236
|
+
clean_url = urlunparse((
|
|
237
|
+
parsed.scheme, netloc, parsed.path,
|
|
238
|
+
parsed.params, parsed.query, parsed.fragment,
|
|
239
|
+
))
|
|
240
|
+
else:
|
|
241
|
+
clean_url = url
|
|
242
|
+
|
|
243
|
+
# Check for ?token= query param
|
|
244
|
+
if not token:
|
|
245
|
+
query_params = parse_qs(parsed.query)
|
|
246
|
+
token_values = query_params.get("token")
|
|
247
|
+
if token_values:
|
|
248
|
+
token = token_values[0]
|
|
249
|
+
# Strip token param from the URL
|
|
250
|
+
from urllib.parse import urlencode
|
|
251
|
+
remaining = {k: v[0] for k, v in query_params.items() if k != "token"}
|
|
252
|
+
new_query = urlencode(remaining) if remaining else ""
|
|
253
|
+
reparsed = urlparse(clean_url)
|
|
254
|
+
from urllib.parse import urlunparse
|
|
255
|
+
clean_url = urlunparse((
|
|
256
|
+
reparsed.scheme, reparsed.netloc, reparsed.path,
|
|
257
|
+
reparsed.params, new_query, reparsed.fragment,
|
|
258
|
+
))
|
|
259
|
+
|
|
260
|
+
return CollectorURLConfig(url=clean_url, token=token)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _parse_opensearch_url_to_config(url: str) -> OpenSearchURLConfig:
|
|
264
|
+
"""Parse a standard http(s) URL into OpenSearchURLConfig."""
|
|
265
|
+
result = _parse_opensearch_url(url)
|
|
266
|
+
if result is None:
|
|
267
|
+
raise URLParseError(f"Invalid URL '{url}'")
|
|
268
|
+
scheme, host, port, user, password, index = result
|
|
269
|
+
return OpenSearchURLConfig(
|
|
270
|
+
scheme=scheme, host=host, port=port,
|
|
271
|
+
user=user, password=password, index=index,
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
|
|
121
275
|
def _parse_opensearch_url(url: str):
|
|
122
276
|
"""Parse DEVLOGS_OPENSEARCH_URL into components.
|
|
123
277
|
|
|
124
278
|
Supports format: https://user:pass@host:port/index
|
|
279
|
+
Also supports: opensearchs://user:pass@host:port/index (TLS)
|
|
280
|
+
opensearch://user:pass@host:port/index (non-TLS)
|
|
125
281
|
Returns: (scheme, host, port, user, pass, index) or None if no URL
|
|
126
282
|
Raises: URLParseError if URL is malformed
|
|
127
283
|
"""
|
|
128
284
|
if not url:
|
|
129
285
|
return None
|
|
130
|
-
|
|
286
|
+
|
|
287
|
+
# Handle opensearchs:// (TLS) and opensearch:// (non-TLS) schemes
|
|
288
|
+
if url.startswith("opensearchs://"):
|
|
289
|
+
actual_url = "https://" + url[len("opensearchs://"):]
|
|
290
|
+
force_scheme = "https"
|
|
291
|
+
elif url.startswith("opensearch://"):
|
|
292
|
+
actual_url = "http://" + url[len("opensearch://"):]
|
|
293
|
+
force_scheme = "http"
|
|
294
|
+
else:
|
|
295
|
+
actual_url = url
|
|
296
|
+
force_scheme = None
|
|
297
|
+
|
|
298
|
+
parsed = urlparse(actual_url)
|
|
131
299
|
|
|
132
300
|
# Validate scheme
|
|
133
301
|
if parsed.scheme and parsed.scheme not in ("http", "https"):
|
|
@@ -137,7 +305,7 @@ def _parse_opensearch_url(url: str):
|
|
|
137
305
|
if not parsed.hostname:
|
|
138
306
|
raise URLParseError(f"Invalid URL '{url}': missing hostname")
|
|
139
307
|
|
|
140
|
-
scheme = parsed.scheme or "http"
|
|
308
|
+
scheme = force_scheme or parsed.scheme or "http"
|
|
141
309
|
host = parsed.hostname
|
|
142
310
|
port = parsed.port or (443 if scheme == "https" else 9200)
|
|
143
311
|
# URL-decode username and password since urlparse doesn't do this automatically
|
|
@@ -227,6 +395,15 @@ class DevlogsConfig:
|
|
|
227
395
|
self.collector_workers = int(_getenv("DEVLOGS_COLLECTOR_WORKERS", "1"))
|
|
228
396
|
self.collector_log_level = _getenv("DEVLOGS_COLLECTOR_LOG_LEVEL", "info")
|
|
229
397
|
|
|
398
|
+
@property
|
|
399
|
+
def url_mode(self) -> str:
|
|
400
|
+
"""Return 'collector' if DEVLOGS_URL is set, 'opensearch' if OpenSearch is configured, else 'none'."""
|
|
401
|
+
if self.collector_url:
|
|
402
|
+
return "collector"
|
|
403
|
+
if self.has_opensearch_config():
|
|
404
|
+
return "opensearch"
|
|
405
|
+
return "none"
|
|
406
|
+
|
|
230
407
|
def has_opensearch_config(self) -> bool:
|
|
231
408
|
"""Check if OpenSearch admin connection is configured."""
|
|
232
409
|
return bool(
|
|
@@ -257,8 +434,27 @@ def set_dotenv_path(path: str):
|
|
|
257
434
|
|
|
258
435
|
|
|
259
436
|
def set_url(url: str):
|
|
260
|
-
"""Set the
|
|
261
|
-
|
|
437
|
+
"""Set the URL, auto-detecting whether it's a collector or OpenSearch URL.
|
|
438
|
+
|
|
439
|
+
Uses parse_url() to detect the URL type:
|
|
440
|
+
- Collector URLs → sets DEVLOGS_URL
|
|
441
|
+
- OpenSearch URLs → sets DEVLOGS_OPENSEARCH_URL
|
|
442
|
+
"""
|
|
443
|
+
if not url:
|
|
444
|
+
return
|
|
445
|
+
try:
|
|
446
|
+
parsed = parse_url(url)
|
|
447
|
+
except URLParseError:
|
|
448
|
+
# Fall back to legacy behavior
|
|
449
|
+
os.environ["DEVLOGS_OPENSEARCH_URL"] = url
|
|
450
|
+
return
|
|
451
|
+
|
|
452
|
+
if isinstance(parsed, CollectorURLConfig):
|
|
453
|
+
# Reconstruct the full collector URL for DEVLOGS_URL
|
|
454
|
+
os.environ["DEVLOGS_URL"] = url
|
|
455
|
+
else:
|
|
456
|
+
# OpenSearch URL - reconstruct with the correct scheme for _parse_opensearch_url
|
|
457
|
+
os.environ["DEVLOGS_OPENSEARCH_URL"] = url
|
|
262
458
|
|
|
263
459
|
def load_config() -> DevlogsConfig:
|
|
264
460
|
"""Return a config object with all settings loaded."""
|
|
@@ -13,29 +13,42 @@ from .handler import OpenSearchHandler
|
|
|
13
13
|
def run_demo(
|
|
14
14
|
duration: int,
|
|
15
15
|
count: int,
|
|
16
|
-
require_opensearch,
|
|
16
|
+
require_opensearch=None,
|
|
17
|
+
collector_url: str = None,
|
|
17
18
|
):
|
|
18
19
|
"""Generate demo logs to illustrate devlogs capabilities."""
|
|
19
20
|
cfg = load_config()
|
|
20
21
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
22
|
+
if collector_url:
|
|
23
|
+
# Collector mode
|
|
24
|
+
typer.echo("=== DevLogs Demo (Collector Mode) ===\n")
|
|
25
|
+
typer.echo(f" Collector URL: {collector_url}")
|
|
26
|
+
typer.echo("")
|
|
27
|
+
|
|
28
|
+
handler = OpenSearchHandler(
|
|
29
|
+
level=logging.DEBUG,
|
|
30
|
+
collector_url=collector_url,
|
|
31
|
+
)
|
|
32
|
+
else:
|
|
33
|
+
# OpenSearch mode
|
|
34
|
+
typer.echo("=== DevLogs Demo ===\n")
|
|
35
|
+
typer.echo("Configuration loaded from .env:")
|
|
36
|
+
typer.echo(f" DEVLOGS_OPENSEARCH_HOST: {cfg.opensearch_host}")
|
|
37
|
+
typer.echo(f" DEVLOGS_OPENSEARCH_PORT: {cfg.opensearch_port}")
|
|
38
|
+
typer.echo(f" DEVLOGS_OPENSEARCH_USER: {cfg.opensearch_user}")
|
|
39
|
+
typer.echo(f" DEVLOGS_OPENSEARCH_PASS: {'*' * len(cfg.opensearch_pass)}")
|
|
40
|
+
typer.echo(f" DEVLOGS_INDEX: {cfg.index}")
|
|
41
|
+
typer.echo(f" DEVLOGS_RETENTION_DEBUG: {cfg.retention_debug_hours}h")
|
|
42
|
+
typer.echo("")
|
|
43
|
+
|
|
44
|
+
# Check OpenSearch connection and index
|
|
45
|
+
client, cfg = require_opensearch()
|
|
46
|
+
handler = OpenSearchHandler(
|
|
47
|
+
level=logging.DEBUG,
|
|
48
|
+
opensearch_client=client,
|
|
49
|
+
index_name=cfg.index,
|
|
50
|
+
)
|
|
51
|
+
|
|
39
52
|
handler.setFormatter(logging.Formatter("%(message)s"))
|
|
40
53
|
|
|
41
54
|
logger = logging.getLogger("devlogs.demo")
|