devlogs 2.3.0__tar.gz → 2.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {devlogs-2.3.0/src/devlogs.egg-info → devlogs-2.3.2}/PKG-INFO +3 -1
- {devlogs-2.3.0 → devlogs-2.3.2}/pyproject.toml +4 -1
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/_version_static.py +1 -1
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/cli.py +148 -1
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/config.py +61 -4
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/loki/queries.py +9 -4
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/mcp/server.py +17 -2
- devlogs-2.3.2/src/devlogs/proxy/__init__.py +0 -0
- devlogs-2.3.2/src/devlogs/proxy/server.py +163 -0
- {devlogs-2.3.0 → devlogs-2.3.2/src/devlogs.egg-info}/PKG-INFO +3 -1
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs.egg-info/SOURCES.txt +3 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs.egg-info/requires.txt +3 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_cli.py +44 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_config.py +67 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_context.py +3 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_handler.py +1 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_indexing.py +4 -0
- devlogs-2.3.2/tests/test_proxy_server.py +265 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/LICENSE +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/MANIFEST.in +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/README.md +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/setup.cfg +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/__init__.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/__main__.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/build_info.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/collector/__init__.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/collector/auth.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/collector/cli.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/collector/errors.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/collector/forwarder.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/collector/ingestor.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/collector/loki_plugin.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/collector/plugins.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/collector/schema.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/collector/server.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/context.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/demo.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/devlogs_client.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/formatting.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/handler.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/jenkins/__init__.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/jenkins/cli.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/jenkins/core.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/levels.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/loki/__init__.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/mcp/__init__.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/opensearch/__init__.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/opensearch/client.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/opensearch/indexing.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/opensearch/mappings.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/opensearch/queries.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/retention.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/scrub.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/time_utils.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/version.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/web/__init__.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/web/server.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/web/static/devlogs.css +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/web/static/devlogs.js +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/web/static/index.html +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs/wrapper.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs.egg-info/dependency_links.txt +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs.egg-info/entry_points.txt +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/src/devlogs.egg-info/top_level.txt +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_build_info.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_collector_auth.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_collector_config.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_collector_plugins.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_collector_schema.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_collector_server.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_devlogs_client.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_formatting.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_levels.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_mappings.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_mcp_server.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_opensearch_client.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_opensearch_queries.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_retention.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_scrub.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_time_utils.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_url_parsing.py +0 -0
- {devlogs-2.3.0 → devlogs-2.3.2}/tests/test_web.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: devlogs
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.2
|
|
4
4
|
Summary: Developer-focused logging library for Python with OpenSearch integration.
|
|
5
5
|
Author-email: Dan Driscoll <dan@thedandriscoll.org>
|
|
6
6
|
License: MIT License
|
|
@@ -39,6 +39,8 @@ Provides-Extra: dev
|
|
|
39
39
|
Requires-Dist: httpx>=0.26.0; extra == "dev"
|
|
40
40
|
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
|
41
41
|
Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev"
|
|
42
|
+
Provides-Extra: proxy
|
|
43
|
+
Requires-Dist: aiohttp>=3.9.0; extra == "proxy"
|
|
42
44
|
Dynamic: license-file
|
|
43
45
|
|
|
44
46
|
# devlogs
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "devlogs"
|
|
7
|
-
version = "2.3.
|
|
7
|
+
version = "2.3.2"
|
|
8
8
|
description = "Developer-focused logging library for Python with OpenSearch integration."
|
|
9
9
|
requires-python = ">=3.11"
|
|
10
10
|
readme = "README.md"
|
|
@@ -26,6 +26,9 @@ dev = [
|
|
|
26
26
|
"pytest>=8.0.0",
|
|
27
27
|
"pytest-asyncio>=0.23.0",
|
|
28
28
|
]
|
|
29
|
+
proxy = [
|
|
30
|
+
"aiohttp>=3.9.0",
|
|
31
|
+
]
|
|
29
32
|
|
|
30
33
|
[project.scripts]
|
|
31
34
|
devlogs = "devlogs.wrapper:main"
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
# AUTO-GENERATED at build time — do not edit or commit
|
|
2
|
-
__version__ = "2.3.
|
|
2
|
+
__version__ = "2.3.2"
|
|
@@ -10,7 +10,7 @@ import click
|
|
|
10
10
|
import typer
|
|
11
11
|
from pathlib import Path
|
|
12
12
|
|
|
13
|
-
from .config import load_config, set_dotenv_path, set_url, URLParseError, _parse_opensearch_url, parse_url, CollectorURLConfig
|
|
13
|
+
from .config import load_config, set_dotenv_path, set_url, URLParseError, _parse_opensearch_url, parse_url, CollectorURLConfig, LokiURLConfig
|
|
14
14
|
from .formatting import format_timestamp, all_entries_today, format_entry_text
|
|
15
15
|
from .opensearch.client import (
|
|
16
16
|
get_opensearch_client,
|
|
@@ -313,6 +313,12 @@ def init(
|
|
|
313
313
|
automatically migrate data from v1 to v2 schema.
|
|
314
314
|
"""
|
|
315
315
|
_apply_common_options(env, url)
|
|
316
|
+
|
|
317
|
+
cfg = load_config()
|
|
318
|
+
if cfg.is_loki:
|
|
319
|
+
typer.echo("Loki backends do not require initialization. No action needed.")
|
|
320
|
+
return
|
|
321
|
+
|
|
316
322
|
client, cfg = require_opensearch(check_idx=False)
|
|
317
323
|
|
|
318
324
|
# Check existing index schema
|
|
@@ -780,6 +786,123 @@ def diagnose(
|
|
|
780
786
|
raise typer.Exit(1)
|
|
781
787
|
|
|
782
788
|
|
|
789
|
+
def _format_loki_entries(entries, utc=False, jsonl=False):
|
|
790
|
+
"""Format and print Loki log entries."""
|
|
791
|
+
if jsonl:
|
|
792
|
+
for entry in entries:
|
|
793
|
+
typer.echo(json.dumps(entry, default=str))
|
|
794
|
+
else:
|
|
795
|
+
use_color = sys.stdout.isatty()
|
|
796
|
+
normalized = []
|
|
797
|
+
for e in entries:
|
|
798
|
+
normalized.append({
|
|
799
|
+
"timestamp": e.get("timestamp") or "",
|
|
800
|
+
"level": e.get("level") or "",
|
|
801
|
+
"area": e.get("area") or "",
|
|
802
|
+
"component": e.get("component") or "",
|
|
803
|
+
"operation_id": e.get("operation_id") or "",
|
|
804
|
+
"message": e.get("message") or "",
|
|
805
|
+
"fields": e.get("fields"),
|
|
806
|
+
})
|
|
807
|
+
omit_date = all_entries_today(normalized, use_utc=utc)
|
|
808
|
+
for doc in normalized:
|
|
809
|
+
line = format_entry_text(
|
|
810
|
+
doc, use_utc=utc, omit_date=omit_date,
|
|
811
|
+
color=use_color, format_features_fn=_format_features,
|
|
812
|
+
)
|
|
813
|
+
typer.echo(line)
|
|
814
|
+
|
|
815
|
+
|
|
816
|
+
def _tail_loki(cfg, application=None, operation_id=None, area=None,
|
|
817
|
+
component=None, level=None, since=None, limit=20, follow=False,
|
|
818
|
+
utc=False, jsonl=False):
|
|
819
|
+
"""Tail logs from a Loki backend."""
|
|
820
|
+
from .loki.queries import tail as loki_tail
|
|
821
|
+
|
|
822
|
+
effective_app = application or cfg.application
|
|
823
|
+
if not effective_app:
|
|
824
|
+
typer.echo(typer.style(
|
|
825
|
+
"Error: --application is required for Loki backends",
|
|
826
|
+
fg=typer.colors.RED
|
|
827
|
+
), err=True)
|
|
828
|
+
raise typer.Exit(1)
|
|
829
|
+
|
|
830
|
+
first_poll = True
|
|
831
|
+
while True:
|
|
832
|
+
try:
|
|
833
|
+
entries = loki_tail(
|
|
834
|
+
loki_url=cfg.loki_url,
|
|
835
|
+
app=effective_app,
|
|
836
|
+
level=level,
|
|
837
|
+
component=component,
|
|
838
|
+
since=since or "10m",
|
|
839
|
+
limit=limit,
|
|
840
|
+
token=cfg.loki_token,
|
|
841
|
+
)
|
|
842
|
+
except Exception as e:
|
|
843
|
+
typer.echo(typer.style(
|
|
844
|
+
f"Error: {type(e).__name__}: {e}",
|
|
845
|
+
fg=typer.colors.RED
|
|
846
|
+
), err=True)
|
|
847
|
+
raise typer.Exit(1)
|
|
848
|
+
|
|
849
|
+
if first_poll and not entries:
|
|
850
|
+
typer.echo(typer.style("No logs found.", dim=True), err=True)
|
|
851
|
+
first_poll = False
|
|
852
|
+
|
|
853
|
+
_format_loki_entries(entries, utc=utc, jsonl=jsonl)
|
|
854
|
+
|
|
855
|
+
if not follow:
|
|
856
|
+
break
|
|
857
|
+
time.sleep(2)
|
|
858
|
+
|
|
859
|
+
|
|
860
|
+
def _search_loki(cfg, q="", application=None, area=None, component=None,
|
|
861
|
+
level=None, operation_id=None, since=None, limit=50, follow=False,
|
|
862
|
+
utc=False, jsonl=False):
|
|
863
|
+
"""Search logs from a Loki backend."""
|
|
864
|
+
from .loki.queries import search as loki_search
|
|
865
|
+
|
|
866
|
+
effective_app = application or cfg.application
|
|
867
|
+
if not effective_app:
|
|
868
|
+
typer.echo(typer.style(
|
|
869
|
+
"Error: --application is required for Loki backends",
|
|
870
|
+
fg=typer.colors.RED
|
|
871
|
+
), err=True)
|
|
872
|
+
raise typer.Exit(1)
|
|
873
|
+
|
|
874
|
+
first_poll = True
|
|
875
|
+
while True:
|
|
876
|
+
try:
|
|
877
|
+
entries = loki_search(
|
|
878
|
+
loki_url=cfg.loki_url,
|
|
879
|
+
app=effective_app,
|
|
880
|
+
level=level,
|
|
881
|
+
component=component,
|
|
882
|
+
area=area,
|
|
883
|
+
start=since,
|
|
884
|
+
limit=limit,
|
|
885
|
+
filter_text=q or None,
|
|
886
|
+
token=cfg.loki_token,
|
|
887
|
+
)
|
|
888
|
+
except Exception as e:
|
|
889
|
+
typer.echo(typer.style(
|
|
890
|
+
f"Error: {type(e).__name__}: {e}",
|
|
891
|
+
fg=typer.colors.RED
|
|
892
|
+
), err=True)
|
|
893
|
+
raise typer.Exit(1)
|
|
894
|
+
|
|
895
|
+
if first_poll and not entries:
|
|
896
|
+
typer.echo(typer.style("No logs found.", dim=True), err=True)
|
|
897
|
+
first_poll = False
|
|
898
|
+
|
|
899
|
+
_format_loki_entries(entries, utc=utc, jsonl=jsonl)
|
|
900
|
+
|
|
901
|
+
if not follow:
|
|
902
|
+
break
|
|
903
|
+
time.sleep(2)
|
|
904
|
+
|
|
905
|
+
|
|
783
906
|
@app.command()
|
|
784
907
|
def tail(
|
|
785
908
|
operation_id: str = typer.Option(None, "--operation", "-o"),
|
|
@@ -801,6 +924,14 @@ def tail(
|
|
|
801
924
|
import traceback
|
|
802
925
|
|
|
803
926
|
_apply_common_options(env, url)
|
|
927
|
+
|
|
928
|
+
cfg = load_config()
|
|
929
|
+
if cfg.is_loki:
|
|
930
|
+
_tail_loki(cfg, application=application, operation_id=operation_id, area=area,
|
|
931
|
+
component=component, level=level, since=since, limit=limit, follow=follow,
|
|
932
|
+
utc=utc, jsonl=jsonl)
|
|
933
|
+
return
|
|
934
|
+
|
|
804
935
|
client, cfg = require_opensearch()
|
|
805
936
|
|
|
806
937
|
def _verbose_echo(message, color=typer.colors.BLUE):
|
|
@@ -982,6 +1113,14 @@ def search(
|
|
|
982
1113
|
import urllib.error
|
|
983
1114
|
|
|
984
1115
|
_apply_common_options(env, url)
|
|
1116
|
+
|
|
1117
|
+
cfg = load_config()
|
|
1118
|
+
if cfg.is_loki:
|
|
1119
|
+
_search_loki(cfg, q=q, application=application, area=area,
|
|
1120
|
+
component=component, level=level, operation_id=operation_id,
|
|
1121
|
+
since=since, limit=limit, follow=follow, utc=utc, jsonl=jsonl)
|
|
1122
|
+
return
|
|
1123
|
+
|
|
985
1124
|
client, cfg = require_opensearch()
|
|
986
1125
|
effective_application = application or cfg.application
|
|
987
1126
|
search_after = None
|
|
@@ -1094,6 +1233,14 @@ def last_error(
|
|
|
1094
1233
|
import urllib.error
|
|
1095
1234
|
|
|
1096
1235
|
_apply_common_options(env, url)
|
|
1236
|
+
|
|
1237
|
+
cfg = load_config()
|
|
1238
|
+
if cfg.is_loki:
|
|
1239
|
+
_search_loki(cfg, q=q, application=application, area=area,
|
|
1240
|
+
component=component, level="error", operation_id=operation_id,
|
|
1241
|
+
since=since, limit=limit, follow=False, utc=utc, jsonl=False)
|
|
1242
|
+
return
|
|
1243
|
+
|
|
1097
1244
|
client, cfg = require_opensearch()
|
|
1098
1245
|
effective_application = application or cfg.application
|
|
1099
1246
|
|
|
@@ -129,6 +129,13 @@ class CollectorURLConfig:
|
|
|
129
129
|
token: Optional[str] = None # Bearer token
|
|
130
130
|
|
|
131
131
|
|
|
132
|
+
@dataclass
|
|
133
|
+
class LokiURLConfig:
|
|
134
|
+
"""Parsed Loki URL configuration (loki:// or lokis:// scheme)."""
|
|
135
|
+
url: str # Base HTTP(S) URL for Loki API (no credentials)
|
|
136
|
+
token: Optional[str] = None # Bearer token
|
|
137
|
+
|
|
138
|
+
|
|
132
139
|
@dataclass
|
|
133
140
|
class OpenSearchURLConfig:
|
|
134
141
|
"""Parsed OpenSearch URL configuration."""
|
|
@@ -142,19 +149,24 @@ class OpenSearchURLConfig:
|
|
|
142
149
|
|
|
143
150
|
|
|
144
151
|
def parse_url(url: str):
|
|
145
|
-
"""Parse a URL and return a CollectorURLConfig or OpenSearchURLConfig.
|
|
152
|
+
"""Parse a URL and return a CollectorURLConfig, LokiURLConfig, or OpenSearchURLConfig.
|
|
146
153
|
|
|
147
154
|
Detection logic:
|
|
155
|
+
- loki:// (non-TLS) or lokis:// (TLS) → Loki
|
|
148
156
|
- opensearchs:// (TLS) or opensearch:// (non-TLS) → OpenSearch
|
|
149
157
|
- https:// or http:// with both user+pass → legacy OpenSearch (deprecation warning)
|
|
150
158
|
- https:// or http:// with token-only or ?token= → Collector
|
|
151
159
|
|
|
152
|
-
Returns: CollectorURLConfig or OpenSearchURLConfig
|
|
160
|
+
Returns: CollectorURLConfig, LokiURLConfig, or OpenSearchURLConfig
|
|
153
161
|
Raises: URLParseError if URL is malformed
|
|
154
162
|
"""
|
|
155
163
|
if not url:
|
|
156
164
|
raise URLParseError("Empty URL")
|
|
157
165
|
|
|
166
|
+
# Check for loki:// or lokis:// schemes
|
|
167
|
+
if url.startswith("lokis://") or url.startswith("loki://"):
|
|
168
|
+
return _parse_loki_url(url)
|
|
169
|
+
|
|
158
170
|
# Check for opensearch:// or opensearchs:// schemes
|
|
159
171
|
if url.startswith("opensearchs://") or url.startswith("opensearch://"):
|
|
160
172
|
return _parse_opensearch_scheme_url(url)
|
|
@@ -188,6 +200,39 @@ def parse_url(url: str):
|
|
|
188
200
|
return _parse_collector_url_config(url)
|
|
189
201
|
|
|
190
202
|
|
|
203
|
+
def _parse_loki_url(url: str) -> LokiURLConfig:
|
|
204
|
+
"""Parse loki:// (non-TLS) or lokis:// (TLS) URL into LokiURLConfig."""
|
|
205
|
+
if url.startswith("lokis://"):
|
|
206
|
+
transport_scheme = "https"
|
|
207
|
+
parse_url_str = "https://" + url[len("lokis://"):]
|
|
208
|
+
else:
|
|
209
|
+
transport_scheme = "http"
|
|
210
|
+
parse_url_str = "http://" + url[len("loki://"):]
|
|
211
|
+
|
|
212
|
+
parsed = urlparse(parse_url_str)
|
|
213
|
+
|
|
214
|
+
if not parsed.hostname:
|
|
215
|
+
raise URLParseError(f"Invalid URL '{url}': missing hostname")
|
|
216
|
+
|
|
217
|
+
token = None
|
|
218
|
+
if parsed.username and not parsed.password:
|
|
219
|
+
token = unquote(parsed.username)
|
|
220
|
+
|
|
221
|
+
# Rebuild clean URL without credentials
|
|
222
|
+
if parsed.port:
|
|
223
|
+
netloc = f"{parsed.hostname}:{parsed.port}"
|
|
224
|
+
else:
|
|
225
|
+
netloc = parsed.hostname or ""
|
|
226
|
+
|
|
227
|
+
from urllib.parse import urlunparse
|
|
228
|
+
clean_url = urlunparse((
|
|
229
|
+
transport_scheme, netloc, parsed.path,
|
|
230
|
+
parsed.params, parsed.query, parsed.fragment,
|
|
231
|
+
))
|
|
232
|
+
|
|
233
|
+
return LokiURLConfig(url=clean_url, token=token)
|
|
234
|
+
|
|
235
|
+
|
|
191
236
|
def _parse_opensearch_scheme_url(url: str) -> OpenSearchURLConfig:
|
|
192
237
|
"""Parse opensearchs:// (TLS) or opensearch:// (non-TLS) URL into OpenSearchURLConfig."""
|
|
193
238
|
# Determine the transport scheme
|
|
@@ -342,6 +387,8 @@ class DevlogsConfig:
|
|
|
342
387
|
# DEVLOGS_URL is the standard env var. It auto-detects collector vs OpenSearch
|
|
343
388
|
# via parse_url(). DEVLOGS_OPENSEARCH_URL is a legacy alias for OpenSearch URLs.
|
|
344
389
|
self.collector_url = ""
|
|
390
|
+
self.loki_url = None # Loki base URL (set when loki:// or lokis:// scheme used)
|
|
391
|
+
self.loki_token = None # Loki auth token
|
|
345
392
|
self.application = None
|
|
346
393
|
opensearch_url_config = None # result from _parse_opensearch_url if found
|
|
347
394
|
|
|
@@ -351,7 +398,10 @@ class DevlogsConfig:
|
|
|
351
398
|
if devlogs_url:
|
|
352
399
|
try:
|
|
353
400
|
parsed = parse_url(devlogs_url)
|
|
354
|
-
if isinstance(parsed,
|
|
401
|
+
if isinstance(parsed, LokiURLConfig):
|
|
402
|
+
self.loki_url = parsed.url
|
|
403
|
+
self.loki_token = parsed.token
|
|
404
|
+
elif isinstance(parsed, CollectorURLConfig):
|
|
355
405
|
self.collector_url = devlogs_url
|
|
356
406
|
else:
|
|
357
407
|
# OpenSearch URL via DEVLOGS_URL
|
|
@@ -427,9 +477,16 @@ class DevlogsConfig:
|
|
|
427
477
|
self.collector_workers = int(_getenv("DEVLOGS_COLLECTOR_WORKERS", "1"))
|
|
428
478
|
self.collector_log_level = _getenv("DEVLOGS_COLLECTOR_LOG_LEVEL", "info")
|
|
429
479
|
|
|
480
|
+
@property
|
|
481
|
+
def is_loki(self) -> bool:
|
|
482
|
+
"""Return True if a Loki backend is configured."""
|
|
483
|
+
return bool(self.loki_url)
|
|
484
|
+
|
|
430
485
|
@property
|
|
431
486
|
def url_mode(self) -> str:
|
|
432
|
-
"""Return 'collector'
|
|
487
|
+
"""Return 'loki', 'collector', 'opensearch', or 'none'."""
|
|
488
|
+
if self.loki_url:
|
|
489
|
+
return "loki"
|
|
433
490
|
if self.collector_url:
|
|
434
491
|
return "collector"
|
|
435
492
|
if self.has_opensearch_config():
|
|
@@ -110,12 +110,14 @@ def _parse_time_param(value: Optional[str]) -> Optional[datetime]:
|
|
|
110
110
|
# HTTP helpers
|
|
111
111
|
# ---------------------------------------------------------------------------
|
|
112
112
|
|
|
113
|
-
def _loki_get(loki_url: str, path: str, params: Dict[str, str]) -> Dict[str, Any]:
|
|
113
|
+
def _loki_get(loki_url: str, path: str, params: Dict[str, str], token: Optional[str] = None) -> Dict[str, Any]:
|
|
114
114
|
"""Execute a GET request against the Loki HTTP API."""
|
|
115
115
|
base = loki_url.rstrip("/")
|
|
116
116
|
qs = urllib.parse.urlencode(params)
|
|
117
117
|
url = f"{base}{path}?{qs}"
|
|
118
118
|
req = urllib.request.Request(url, method="GET")
|
|
119
|
+
if token:
|
|
120
|
+
req.add_header("Authorization", f"Bearer {token}")
|
|
119
121
|
try:
|
|
120
122
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
121
123
|
return json.loads(resp.read().decode("utf-8"))
|
|
@@ -185,6 +187,7 @@ def search(
|
|
|
185
187
|
end: Optional[str] = None,
|
|
186
188
|
limit: int = 100,
|
|
187
189
|
filter_text: Optional[str] = None,
|
|
190
|
+
token: Optional[str] = None,
|
|
188
191
|
) -> List[Dict[str, Any]]:
|
|
189
192
|
"""Search log entries in a time range.
|
|
190
193
|
|
|
@@ -229,7 +232,7 @@ def search(
|
|
|
229
232
|
"end": str(_to_ns(end_dt)),
|
|
230
233
|
"limit": str(limit),
|
|
231
234
|
"direction": "backward",
|
|
232
|
-
})
|
|
235
|
+
}, token=token)
|
|
233
236
|
|
|
234
237
|
entries = _parse_log_streams(data)
|
|
235
238
|
# Sort by Loki timestamp descending (most recent first)
|
|
@@ -244,6 +247,7 @@ def tail(
|
|
|
244
247
|
component: Optional[str] = None,
|
|
245
248
|
since: Optional[str] = None,
|
|
246
249
|
limit: int = 50,
|
|
250
|
+
token: Optional[str] = None,
|
|
247
251
|
) -> List[Dict[str, Any]]:
|
|
248
252
|
"""Get the most recent log entries (non-streaming snapshot).
|
|
249
253
|
|
|
@@ -276,7 +280,7 @@ def tail(
|
|
|
276
280
|
"end": str(_to_ns(now)),
|
|
277
281
|
"limit": str(limit),
|
|
278
282
|
"direction": "backward",
|
|
279
|
-
})
|
|
283
|
+
}, token=token)
|
|
280
284
|
|
|
281
285
|
entries = _parse_log_streams(data)
|
|
282
286
|
entries.sort(key=lambda e: e.get("_loki_ts_ns", "0"), reverse=True)
|
|
@@ -290,6 +294,7 @@ def count_over_time(
|
|
|
290
294
|
group_by: Optional[List[str]] = None,
|
|
291
295
|
start: Optional[str] = None,
|
|
292
296
|
end: Optional[str] = None,
|
|
297
|
+
token: Optional[str] = None,
|
|
293
298
|
) -> List[Dict[str, Any]]:
|
|
294
299
|
"""Aggregate log counts over a time interval.
|
|
295
300
|
|
|
@@ -327,6 +332,6 @@ def count_over_time(
|
|
|
327
332
|
"start": str(_to_ns(start_dt)),
|
|
328
333
|
"end": str(_to_ns(end_dt)),
|
|
329
334
|
"step": interval,
|
|
330
|
-
})
|
|
335
|
+
}, token=token)
|
|
331
336
|
|
|
332
337
|
return _parse_metric_matrix(data)
|
|
@@ -115,8 +115,20 @@ def _error_response(message: str, error_type: str = "Error") -> list[types.TextC
|
|
|
115
115
|
|
|
116
116
|
|
|
117
117
|
def _get_loki_url() -> str | None:
|
|
118
|
-
"""Return
|
|
119
|
-
|
|
118
|
+
"""Return Loki URL from LOKI_URL env var or auto-detected from DEVLOGS_URL."""
|
|
119
|
+
url = os.environ.get("LOKI_URL")
|
|
120
|
+
if url:
|
|
121
|
+
return url
|
|
122
|
+
cfg = load_config()
|
|
123
|
+
if cfg.is_loki:
|
|
124
|
+
return cfg.loki_url
|
|
125
|
+
return None
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _get_loki_token() -> str | None:
|
|
129
|
+
"""Return Loki auth token from config, if available."""
|
|
130
|
+
cfg = load_config()
|
|
131
|
+
return cfg.loki_token
|
|
120
132
|
|
|
121
133
|
|
|
122
134
|
def _handle_loki_search(arguments: dict) -> list[types.TextContent]:
|
|
@@ -142,6 +154,7 @@ def _handle_loki_search(arguments: dict) -> list[types.TextContent]:
|
|
|
142
154
|
end=arguments.get("until"),
|
|
143
155
|
limit=_coerce_limit(arguments.get("limit"), 50, 100),
|
|
144
156
|
filter_text=arguments.get("query"),
|
|
157
|
+
token=_get_loki_token(),
|
|
145
158
|
)
|
|
146
159
|
return _json_response(
|
|
147
160
|
data={"entries": entries},
|
|
@@ -171,6 +184,7 @@ def _handle_loki_tail(arguments: dict) -> list[types.TextContent]:
|
|
|
171
184
|
component=arguments.get("component"),
|
|
172
185
|
since=arguments.get("since", "10m"),
|
|
173
186
|
limit=_coerce_limit(arguments.get("limit"), 20, 100),
|
|
187
|
+
token=_get_loki_token(),
|
|
174
188
|
)
|
|
175
189
|
return _json_response(
|
|
176
190
|
data={"entries": entries},
|
|
@@ -200,6 +214,7 @@ def _handle_loki_get_log_stats(arguments: dict) -> list[types.TextContent]:
|
|
|
200
214
|
group_by=arguments.get("group_by"),
|
|
201
215
|
start=arguments.get("since"),
|
|
202
216
|
end=arguments.get("until"),
|
|
217
|
+
token=_get_loki_token(),
|
|
203
218
|
)
|
|
204
219
|
return _json_response(
|
|
205
220
|
data={"stats": stats},
|
|
File without changes
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
# Devlogs proxy server
|
|
2
|
+
#
|
|
3
|
+
# Routes external traffic to internal services:
|
|
4
|
+
# POST /ingest/* → Collector (token-in-URL auth, unchanged)
|
|
5
|
+
# GET /query/* → Loki :3100 (Bearer token auth)
|
|
6
|
+
# GET /grafana/* → Grafana :3000 (Bearer token auth)
|
|
7
|
+
#
|
|
8
|
+
# Environment variables:
|
|
9
|
+
# COLLECTOR_URL — Collector base URL (default: http://localhost:8081)
|
|
10
|
+
# LOKI_URL — Loki base URL (default: http://localhost:3100)
|
|
11
|
+
# GRAFANA_URL — Grafana base URL (default: http://localhost:3000)
|
|
12
|
+
# LOKI_ADMIN_TOKEN — Bearer token for /query and /grafana routes
|
|
13
|
+
# PORT — Port to listen on (default: 8080)
|
|
14
|
+
#
|
|
15
|
+
# Run:
|
|
16
|
+
# python -m devlogs.proxy.server
|
|
17
|
+
|
|
18
|
+
import hmac
|
|
19
|
+
import logging
|
|
20
|
+
import os
|
|
21
|
+
import posixpath
|
|
22
|
+
|
|
23
|
+
try:
|
|
24
|
+
from aiohttp import web, ClientSession, ClientTimeout
|
|
25
|
+
except ImportError as e:
|
|
26
|
+
raise ImportError("aiohttp is required: pip install devlogs[proxy]") from e
|
|
27
|
+
|
|
28
|
+
logger = logging.getLogger("devlogs.proxy")
|
|
29
|
+
|
|
30
|
+
COLLECTOR_URL = os.environ.get("COLLECTOR_URL", "http://localhost:8081").rstrip("/")
|
|
31
|
+
LOKI_URL = os.environ.get("LOKI_URL", "http://localhost:3100").rstrip("/")
|
|
32
|
+
GRAFANA_URL = os.environ.get("GRAFANA_URL", "http://localhost:3000").rstrip("/")
|
|
33
|
+
LOKI_ADMIN_TOKEN = os.environ.get("LOKI_ADMIN_TOKEN", "")
|
|
34
|
+
PORT = int(os.environ.get("PORT", "8080"))
|
|
35
|
+
|
|
36
|
+
_SKIP_HEADERS = frozenset({
|
|
37
|
+
"host", "content-length", "transfer-encoding",
|
|
38
|
+
"x-forwarded-for", "x-forwarded-host", "x-forwarded-proto",
|
|
39
|
+
"forwarded", "x-real-ip", "x-original-url", "x-rewrite-url",
|
|
40
|
+
})
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _proxy_headers(request: web.Request) -> dict:
|
|
44
|
+
return {k: v for k, v in request.headers.items() if k.lower() not in _SKIP_HEADERS}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _build_target(base: str, strip_prefix: str, request: web.Request) -> str:
|
|
48
|
+
path = request.path.removeprefix(strip_prefix) or "/"
|
|
49
|
+
path = posixpath.normpath(path)
|
|
50
|
+
if not path.startswith("/"):
|
|
51
|
+
path = "/" + path
|
|
52
|
+
url = f"{base}{path}"
|
|
53
|
+
if request.query_string:
|
|
54
|
+
url += f"?{request.query_string}"
|
|
55
|
+
return url
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _check_admin_token(request: web.Request) -> bool:
|
|
59
|
+
if not LOKI_ADMIN_TOKEN:
|
|
60
|
+
return False
|
|
61
|
+
return hmac.compare_digest(
|
|
62
|
+
request.headers.get("Authorization", ""),
|
|
63
|
+
f"Bearer {LOKI_ADMIN_TOKEN}",
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
async def handle_ingest(request: web.Request) -> web.Response:
|
|
68
|
+
"""Forward /ingest/* to Collector. Token in URL is passed through; Collector validates it."""
|
|
69
|
+
target = _build_target(COLLECTOR_URL, "/ingest", request)
|
|
70
|
+
body = await request.read()
|
|
71
|
+
|
|
72
|
+
async with request.app["session"].request(
|
|
73
|
+
method=request.method,
|
|
74
|
+
url=target,
|
|
75
|
+
headers=_proxy_headers(request),
|
|
76
|
+
data=body,
|
|
77
|
+
allow_redirects=False,
|
|
78
|
+
) as resp:
|
|
79
|
+
resp_body = await resp.read()
|
|
80
|
+
log_url = target.split("?")[0]
|
|
81
|
+
logger.info("%s /ingest → %s %d", request.method, log_url, resp.status)
|
|
82
|
+
content_type = resp.content_type or "application/octet-stream"
|
|
83
|
+
return web.Response(status=resp.status, body=resp_body, content_type=content_type)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
async def handle_query(request: web.Request) -> web.Response:
|
|
87
|
+
"""Validate Bearer token, strip /query prefix, forward to Loki."""
|
|
88
|
+
if not _check_admin_token(request):
|
|
89
|
+
return web.Response(status=401, text="Unauthorized")
|
|
90
|
+
|
|
91
|
+
target = _build_target(LOKI_URL, "/query", request)
|
|
92
|
+
body = await request.read()
|
|
93
|
+
|
|
94
|
+
async with request.app["session"].request(
|
|
95
|
+
method=request.method,
|
|
96
|
+
url=target,
|
|
97
|
+
headers=_proxy_headers(request),
|
|
98
|
+
data=body,
|
|
99
|
+
allow_redirects=False,
|
|
100
|
+
) as resp:
|
|
101
|
+
resp_body = await resp.read()
|
|
102
|
+
logger.info("%s /query → %s %d", request.method, target, resp.status)
|
|
103
|
+
content_type = resp.content_type or "application/octet-stream"
|
|
104
|
+
return web.Response(status=resp.status, body=resp_body, content_type=content_type)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
async def handle_grafana(request: web.Request) -> web.Response:
|
|
108
|
+
"""Validate Bearer token, strip /grafana prefix, forward to Grafana."""
|
|
109
|
+
if not _check_admin_token(request):
|
|
110
|
+
return web.Response(status=401, text="Unauthorized")
|
|
111
|
+
|
|
112
|
+
target = _build_target(GRAFANA_URL, "/grafana", request)
|
|
113
|
+
# Strip Authorization so Grafana uses its own session mechanism
|
|
114
|
+
headers = {k: v for k, v in _proxy_headers(request).items() if k.lower() != "authorization"}
|
|
115
|
+
body = await request.read()
|
|
116
|
+
|
|
117
|
+
async with request.app["session"].request(
|
|
118
|
+
method=request.method,
|
|
119
|
+
url=target,
|
|
120
|
+
headers=headers,
|
|
121
|
+
data=body,
|
|
122
|
+
allow_redirects=False,
|
|
123
|
+
) as resp:
|
|
124
|
+
resp_body = await resp.read()
|
|
125
|
+
logger.info("%s /grafana → %s %d", request.method, target, resp.status)
|
|
126
|
+
content_type = resp.content_type or "application/octet-stream"
|
|
127
|
+
return web.Response(status=resp.status, body=resp_body, content_type=content_type)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
async def on_startup(app: web.Application) -> None:
|
|
131
|
+
timeout = ClientTimeout(total=30)
|
|
132
|
+
app["session"] = ClientSession(timeout=timeout)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
async def on_cleanup(app: web.Application) -> None:
|
|
136
|
+
await app["session"].close()
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def create_app() -> web.Application:
|
|
140
|
+
app = web.Application(client_max_size=1024 * 1024) # 1 MB
|
|
141
|
+
app.on_startup.append(on_startup)
|
|
142
|
+
app.on_cleanup.append(on_cleanup)
|
|
143
|
+
|
|
144
|
+
app.router.add_route("*", "/ingest", handle_ingest)
|
|
145
|
+
app.router.add_route("*", "/ingest/{path_info:.*}", handle_ingest)
|
|
146
|
+
app.router.add_route("*", "/query", handle_query)
|
|
147
|
+
app.router.add_route("*", "/query/{path_info:.*}", handle_query)
|
|
148
|
+
app.router.add_route("*", "/grafana", handle_grafana)
|
|
149
|
+
app.router.add_route("*", "/grafana/{path_info:.*}", handle_grafana)
|
|
150
|
+
|
|
151
|
+
return app
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
if __name__ == "__main__":
|
|
155
|
+
logging.basicConfig(
|
|
156
|
+
level=logging.INFO,
|
|
157
|
+
format="%(asctime)s %(name)s %(levelname)s %(message)s",
|
|
158
|
+
)
|
|
159
|
+
if not LOKI_ADMIN_TOKEN:
|
|
160
|
+
logger.warning("LOKI_ADMIN_TOKEN is not set — /query and /grafana routes will reject all requests")
|
|
161
|
+
|
|
162
|
+
app = create_app()
|
|
163
|
+
web.run_app(app, host="0.0.0.0", port=PORT)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: devlogs
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.2
|
|
4
4
|
Summary: Developer-focused logging library for Python with OpenSearch integration.
|
|
5
5
|
Author-email: Dan Driscoll <dan@thedandriscoll.org>
|
|
6
6
|
License: MIT License
|
|
@@ -39,6 +39,8 @@ Provides-Extra: dev
|
|
|
39
39
|
Requires-Dist: httpx>=0.26.0; extra == "dev"
|
|
40
40
|
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
|
41
41
|
Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev"
|
|
42
|
+
Provides-Extra: proxy
|
|
43
|
+
Requires-Dist: aiohttp>=3.9.0; extra == "proxy"
|
|
42
44
|
Dynamic: license-file
|
|
43
45
|
|
|
44
46
|
# devlogs
|
|
@@ -47,6 +47,8 @@ src/devlogs/opensearch/client.py
|
|
|
47
47
|
src/devlogs/opensearch/indexing.py
|
|
48
48
|
src/devlogs/opensearch/mappings.py
|
|
49
49
|
src/devlogs/opensearch/queries.py
|
|
50
|
+
src/devlogs/proxy/__init__.py
|
|
51
|
+
src/devlogs/proxy/server.py
|
|
50
52
|
src/devlogs/web/__init__.py
|
|
51
53
|
src/devlogs/web/server.py
|
|
52
54
|
src/devlogs/web/static/devlogs.css
|
|
@@ -70,6 +72,7 @@ tests/test_mappings.py
|
|
|
70
72
|
tests/test_mcp_server.py
|
|
71
73
|
tests/test_opensearch_client.py
|
|
72
74
|
tests/test_opensearch_queries.py
|
|
75
|
+
tests/test_proxy_server.py
|
|
73
76
|
tests/test_retention.py
|
|
74
77
|
tests/test_scrub.py
|
|
75
78
|
tests/test_time_utils.py
|