kafka-viewer 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kafka_viewer-0.1.0/LICENSE +21 -0
- kafka_viewer-0.1.0/PKG-INFO +76 -0
- kafka_viewer-0.1.0/README.md +59 -0
- kafka_viewer-0.1.0/kafka_viewer/__init__.py +3 -0
- kafka_viewer-0.1.0/kafka_viewer/cli.py +29 -0
- kafka_viewer-0.1.0/kafka_viewer/config.py +29 -0
- kafka_viewer-0.1.0/kafka_viewer/kafka_client.py +165 -0
- kafka_viewer-0.1.0/kafka_viewer/ui.py +127 -0
- kafka_viewer-0.1.0/kafka_viewer.egg-info/PKG-INFO +76 -0
- kafka_viewer-0.1.0/kafka_viewer.egg-info/SOURCES.txt +17 -0
- kafka_viewer-0.1.0/kafka_viewer.egg-info/dependency_links.txt +1 -0
- kafka_viewer-0.1.0/kafka_viewer.egg-info/entry_points.txt +2 -0
- kafka_viewer-0.1.0/kafka_viewer.egg-info/requires.txt +11 -0
- kafka_viewer-0.1.0/kafka_viewer.egg-info/top_level.txt +1 -0
- kafka_viewer-0.1.0/pyproject.toml +25 -0
- kafka_viewer-0.1.0/setup.cfg +4 -0
- kafka_viewer-0.1.0/tests/test_cli.py +38 -0
- kafka_viewer-0.1.0/tests/test_config.py +31 -0
- kafka_viewer-0.1.0/tests/test_kafka_client.py +130 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Shashank Yadav
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: kafka-viewer
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A lightweight local Kafka message inspection tool
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Requires-Dist: kafka-python<4,>=3.0.11
|
|
9
|
+
Requires-Dist: streamlit<2,>=1.35
|
|
10
|
+
Provides-Extra: test
|
|
11
|
+
Requires-Dist: pytest<10,>=9.0.3; extra == "test"
|
|
12
|
+
Provides-Extra: build
|
|
13
|
+
Requires-Dist: build<2,>=1; extra == "build"
|
|
14
|
+
Provides-Extra: security
|
|
15
|
+
Requires-Dist: pip-audit<3,>=2.9; extra == "security"
|
|
16
|
+
Dynamic: license-file
|
|
17
|
+
|
|
18
|
+
# kafka-viewer
|
|
19
|
+
|
|
20
|
+
`kafka-viewer` is a lightweight local Streamlit UI for inspecting messages in an unsecured Kafka cluster. Kafka connection details come from a local properties file, and the viewer does not commit consumer offsets.
|
|
21
|
+
|
|
22
|
+
## Installation
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install kafka-viewer
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Configuration
|
|
29
|
+
|
|
30
|
+
Create a local properties file named `kafka-viewer.properties`:
|
|
31
|
+
|
|
32
|
+
```properties
|
|
33
|
+
kafka.bootstrap.servers=localhost:9092
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
The configuration remains local to your machine.
|
|
37
|
+
|
|
38
|
+
## Usage
|
|
39
|
+
|
|
40
|
+
The `--config` option is required:
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
kafka-viewer-unsecured --config /path/to/kafka-viewer.properties
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Windows:
|
|
47
|
+
|
|
48
|
+
```text
|
|
49
|
+
kafka-viewer-unsecured --config C:\path\to\kafka-viewer.properties
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
The UI lets you check Kafka connection status, discover and select topics, enter or generate a temporary consumer group ID, and load:
|
|
53
|
+
|
|
54
|
+
- The latest N messages.
|
|
55
|
+
- Messages from the beginning.
|
|
56
|
+
- Messages from a date/time.
|
|
57
|
+
- Messages within a date/time range.
|
|
58
|
+
|
|
59
|
+
Loaded messages show the partition, offset, timestamp, key, and message value. Messages can be expanded to view the complete message value, including formatted JSON when applicable.
|
|
60
|
+
|
|
61
|
+
## V1 limitations
|
|
62
|
+
|
|
63
|
+
V1 supports unsecured Kafka only. It does not support:
|
|
64
|
+
|
|
65
|
+
- Kafka authentication or security configuration.
|
|
66
|
+
- Avro, Schema Registry, or Protobuf.
|
|
67
|
+
- Publishing messages.
|
|
68
|
+
- Topic or consumer-group administration.
|
|
69
|
+
- Multiple Kafka clusters.
|
|
70
|
+
|
|
71
|
+
## Development
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
python -m pytest
|
|
75
|
+
python -m build
|
|
76
|
+
```
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# kafka-viewer
|
|
2
|
+
|
|
3
|
+
`kafka-viewer` is a lightweight local Streamlit UI for inspecting messages in an unsecured Kafka cluster. Kafka connection details come from a local properties file, and the viewer does not commit consumer offsets.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install kafka-viewer
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Configuration
|
|
12
|
+
|
|
13
|
+
Create a local properties file named `kafka-viewer.properties`:
|
|
14
|
+
|
|
15
|
+
```properties
|
|
16
|
+
kafka.bootstrap.servers=localhost:9092
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
The configuration remains local to your machine.
|
|
20
|
+
|
|
21
|
+
## Usage
|
|
22
|
+
|
|
23
|
+
The `--config` option is required:
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
kafka-viewer-unsecured --config /path/to/kafka-viewer.properties
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Windows:
|
|
30
|
+
|
|
31
|
+
```text
|
|
32
|
+
kafka-viewer-unsecured --config C:\path\to\kafka-viewer.properties
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
The UI lets you check Kafka connection status, discover and select topics, enter or generate a temporary consumer group ID, and load:
|
|
36
|
+
|
|
37
|
+
- The latest N messages.
|
|
38
|
+
- Messages from the beginning.
|
|
39
|
+
- Messages from a date/time.
|
|
40
|
+
- Messages within a date/time range.
|
|
41
|
+
|
|
42
|
+
Loaded messages show the partition, offset, timestamp, key, and message value. Messages can be expanded to view the complete message value, including formatted JSON when applicable.
|
|
43
|
+
|
|
44
|
+
## V1 limitations
|
|
45
|
+
|
|
46
|
+
V1 supports unsecured Kafka only. It does not support:
|
|
47
|
+
|
|
48
|
+
- Kafka authentication or security configuration.
|
|
49
|
+
- Avro, Schema Registry, or Protobuf.
|
|
50
|
+
- Publishing messages.
|
|
51
|
+
- Topic or consumer-group administration.
|
|
52
|
+
- Multiple Kafka clusters.
|
|
53
|
+
|
|
54
|
+
## Development
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
python -m pytest
|
|
58
|
+
python -m build
|
|
59
|
+
```
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import subprocess
|
|
3
|
+
import sys
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from .config import load_properties
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def main() -> None:
|
|
10
|
+
parser = argparse.ArgumentParser(description="Start the unsecured Kafka viewer")
|
|
11
|
+
parser.add_argument("--config", required=True, help="Path to a properties file")
|
|
12
|
+
args = parser.parse_args()
|
|
13
|
+
config_path = Path(args.config)
|
|
14
|
+
load_properties(config_path)
|
|
15
|
+
command = [
|
|
16
|
+
sys.executable,
|
|
17
|
+
"-m",
|
|
18
|
+
"streamlit",
|
|
19
|
+
"run",
|
|
20
|
+
str(Path(__file__).with_name("ui.py")),
|
|
21
|
+
"--",
|
|
22
|
+
"--config",
|
|
23
|
+
str(config_path),
|
|
24
|
+
]
|
|
25
|
+
raise SystemExit(subprocess.call(command))
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
if __name__ == "__main__":
|
|
29
|
+
main()
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class ConfigError(ValueError):
|
|
5
|
+
"""Raised when the viewer configuration is invalid."""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def load_properties(path: str | Path) -> dict[str, str]:
|
|
9
|
+
config_path = Path(path)
|
|
10
|
+
if not config_path.is_file():
|
|
11
|
+
raise FileNotFoundError(f"Configuration file not found: {config_path}")
|
|
12
|
+
|
|
13
|
+
properties: dict[str, str] = {}
|
|
14
|
+
for line_number, raw_line in enumerate(config_path.read_text(encoding="utf-8").splitlines(), 1):
|
|
15
|
+
line = raw_line.strip()
|
|
16
|
+
if not line or line.startswith("#"):
|
|
17
|
+
continue
|
|
18
|
+
if "=" not in line:
|
|
19
|
+
raise ConfigError(f"Invalid configuration line {line_number}: expected key=value")
|
|
20
|
+
key, value = (part.strip() for part in line.split("=", 1))
|
|
21
|
+
if not key:
|
|
22
|
+
raise ConfigError(f"Invalid configuration line {line_number}: empty key")
|
|
23
|
+
properties[key] = value
|
|
24
|
+
|
|
25
|
+
bootstrap_servers = properties.get("kafka.bootstrap.servers", "").strip()
|
|
26
|
+
if not bootstrap_servers:
|
|
27
|
+
raise ConfigError("Missing required property: kafka.bootstrap.servers")
|
|
28
|
+
properties["kafka.bootstrap.servers"] = bootstrap_servers
|
|
29
|
+
return properties
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import uuid
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
from typing import Any, Callable, Iterable
|
|
8
|
+
|
|
9
|
+
from kafka import KafkaConsumer, TopicPartition
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True)
|
|
13
|
+
class Message:
|
|
14
|
+
partition: int
|
|
15
|
+
offset: int
|
|
16
|
+
timestamp: int | None
|
|
17
|
+
key: Any
|
|
18
|
+
value: Any
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def validate_count(value: Any) -> int:
|
|
22
|
+
try:
|
|
23
|
+
count = int(value)
|
|
24
|
+
except (TypeError, ValueError) as exc:
|
|
25
|
+
raise ValueError("Message count must be a positive integer") from exc
|
|
26
|
+
if count <= 0:
|
|
27
|
+
raise ValueError("Message count must be a positive integer")
|
|
28
|
+
return count
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def generate_group_id() -> str:
|
|
32
|
+
return f"kafka-viewer-{uuid.uuid4().hex[:12]}"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _readable(value: Any) -> str:
|
|
36
|
+
if value is None:
|
|
37
|
+
return "<null>"
|
|
38
|
+
if isinstance(value, bytes):
|
|
39
|
+
return value.decode("utf-8", errors="replace")
|
|
40
|
+
return str(value)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def format_value(value: Any) -> str:
|
|
44
|
+
if value is None:
|
|
45
|
+
return "<null>"
|
|
46
|
+
readable = _readable(value)
|
|
47
|
+
try:
|
|
48
|
+
parsed = json.loads(readable)
|
|
49
|
+
except (TypeError, json.JSONDecodeError):
|
|
50
|
+
return readable
|
|
51
|
+
return json.dumps(parsed, indent=2, ensure_ascii=False)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def format_key(value: Any) -> str:
|
|
55
|
+
return _readable(value)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def datetime_to_millis(value: datetime) -> int:
|
|
59
|
+
if value.tzinfo is None:
|
|
60
|
+
value = value.replace(tzinfo=timezone.utc)
|
|
61
|
+
return int(value.timestamp() * 1000)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class KafkaClient:
|
|
65
|
+
def __init__(self, bootstrap_servers: str, consumer_factory: Callable[..., Any] = KafkaConsumer):
|
|
66
|
+
self.bootstrap_servers = bootstrap_servers
|
|
67
|
+
self.consumer_factory = consumer_factory
|
|
68
|
+
|
|
69
|
+
def topics(self) -> set[str]:
|
|
70
|
+
consumer = self.consumer_factory(
|
|
71
|
+
bootstrap_servers=self.bootstrap_servers,
|
|
72
|
+
request_timeout_ms=5000,
|
|
73
|
+
api_version_auto_timeout_ms=3000,
|
|
74
|
+
)
|
|
75
|
+
try:
|
|
76
|
+
return set(consumer.topics())
|
|
77
|
+
finally:
|
|
78
|
+
consumer.close()
|
|
79
|
+
|
|
80
|
+
def load_messages(
|
|
81
|
+
self,
|
|
82
|
+
topic: str,
|
|
83
|
+
group_id: str,
|
|
84
|
+
mode: str,
|
|
85
|
+
count: int,
|
|
86
|
+
start: datetime | None = None,
|
|
87
|
+
end: datetime | None = None,
|
|
88
|
+
) -> list[Message]:
|
|
89
|
+
count = validate_count(count)
|
|
90
|
+
if mode not in {"latest", "beginning", "from_date", "range"}:
|
|
91
|
+
raise ValueError("Unknown loading mode")
|
|
92
|
+
if mode in {"from_date", "range"} and start is None:
|
|
93
|
+
raise ValueError("A start date/time is required")
|
|
94
|
+
if mode == "range" and end is None:
|
|
95
|
+
raise ValueError("An end date/time is required")
|
|
96
|
+
if mode == "range" and datetime_to_millis(start) > datetime_to_millis(end):
|
|
97
|
+
raise ValueError("End date/time must not be before start date/time")
|
|
98
|
+
|
|
99
|
+
consumer = self.consumer_factory(
|
|
100
|
+
bootstrap_servers=self.bootstrap_servers,
|
|
101
|
+
group_id=group_id or None,
|
|
102
|
+
enable_auto_commit=False,
|
|
103
|
+
consumer_timeout_ms=1500,
|
|
104
|
+
request_timeout_ms=5000,
|
|
105
|
+
)
|
|
106
|
+
try:
|
|
107
|
+
partitions = [TopicPartition(topic, number) for number in (consumer.partitions_for_topic(topic) or set())]
|
|
108
|
+
if not partitions:
|
|
109
|
+
return []
|
|
110
|
+
end_offsets = consumer.end_offsets(partitions)
|
|
111
|
+
if mode == "latest":
|
|
112
|
+
starts = {partition: max(0, end_offsets[partition] - count) for partition in partitions}
|
|
113
|
+
elif mode == "beginning":
|
|
114
|
+
starts = consumer.beginning_offsets(partitions)
|
|
115
|
+
else:
|
|
116
|
+
starts = consumer.offsets_for_times({partition: datetime_to_millis(start) for partition in partitions})
|
|
117
|
+
starts = {partition: offset.offset for partition, offset in starts.items() if offset is not None}
|
|
118
|
+
|
|
119
|
+
active = {partition for partition, offset in starts.items() if offset < end_offsets[partition]}
|
|
120
|
+
if not active:
|
|
121
|
+
return []
|
|
122
|
+
consumer.assign(list(active))
|
|
123
|
+
for partition, offset in starts.items():
|
|
124
|
+
if partition not in active:
|
|
125
|
+
continue
|
|
126
|
+
consumer.seek(partition, offset)
|
|
127
|
+
|
|
128
|
+
records: list[Message] = []
|
|
129
|
+
idle_polls = 0
|
|
130
|
+
while active and (mode == "latest" or len(records) < count):
|
|
131
|
+
batch = consumer.poll(timeout_ms=1000)
|
|
132
|
+
if not batch:
|
|
133
|
+
idle_polls += 1
|
|
134
|
+
if idle_polls >= 2:
|
|
135
|
+
break
|
|
136
|
+
continue
|
|
137
|
+
idle_polls = 0
|
|
138
|
+
for record in _records(batch):
|
|
139
|
+
partition = TopicPartition(record.topic, record.partition)
|
|
140
|
+
if record.offset >= end_offsets[partition]:
|
|
141
|
+
continue
|
|
142
|
+
timestamp = record.timestamp
|
|
143
|
+
if mode == "from_date" and (timestamp is None or timestamp < datetime_to_millis(start)):
|
|
144
|
+
continue
|
|
145
|
+
if mode == "range" and (timestamp is None or timestamp < datetime_to_millis(start) or timestamp > datetime_to_millis(end)):
|
|
146
|
+
if timestamp is not None and timestamp > datetime_to_millis(end):
|
|
147
|
+
active.discard(partition)
|
|
148
|
+
continue
|
|
149
|
+
records.append(Message(record.partition, record.offset, timestamp, record.key, record.value))
|
|
150
|
+
if mode == "range" and timestamp is not None and timestamp >= datetime_to_millis(end):
|
|
151
|
+
active.discard(partition)
|
|
152
|
+
if len(records) >= count:
|
|
153
|
+
break
|
|
154
|
+
for partition in list(active):
|
|
155
|
+
if consumer.position(partition) >= end_offsets[partition]:
|
|
156
|
+
active.discard(partition)
|
|
157
|
+
records.sort(key=lambda item: (item.timestamp if item.timestamp is not None else -1, item.partition, item.offset))
|
|
158
|
+
return records[-count:]
|
|
159
|
+
finally:
|
|
160
|
+
consumer.close()
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _records(batch: dict[Any, Iterable[Any]]) -> Iterable[Any]:
|
|
164
|
+
for records in batch.values():
|
|
165
|
+
yield from records
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import sys
|
|
5
|
+
from datetime import datetime, time
|
|
6
|
+
|
|
7
|
+
import streamlit as st
|
|
8
|
+
|
|
9
|
+
from kafka_viewer.config import load_properties
|
|
10
|
+
from kafka_viewer.kafka_client import KafkaClient, format_key, format_value, generate_group_id, validate_count
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _config_path() -> str:
|
|
14
|
+
parser = argparse.ArgumentParser(add_help=False)
|
|
15
|
+
parser.add_argument("--config", default="kafka-viewer.properties")
|
|
16
|
+
return parser.parse_known_args(sys.argv[1:])[0].config
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _date_time(label: str, key: str) -> datetime:
|
|
20
|
+
selected_date = st.date_input(label, key=f"{key}-date")
|
|
21
|
+
selected_time = st.time_input(f"{label} time", value=time(0, 0), key=f"{key}-time")
|
|
22
|
+
return datetime.combine(selected_date, selected_time)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def main() -> None:
|
|
26
|
+
st.set_page_config(page_title="Kafka Viewer", layout="wide")
|
|
27
|
+
st.title("Kafka Viewer")
|
|
28
|
+
config_path = _config_path()
|
|
29
|
+
st.caption(f"Configuration: {config_path}")
|
|
30
|
+
|
|
31
|
+
try:
|
|
32
|
+
properties = load_properties(config_path)
|
|
33
|
+
except (OSError, ValueError) as exc:
|
|
34
|
+
st.error(str(exc))
|
|
35
|
+
return
|
|
36
|
+
|
|
37
|
+
bootstrap_servers = properties["kafka.bootstrap.servers"]
|
|
38
|
+
st.write(f"Broker: `{bootstrap_servers}`")
|
|
39
|
+
client = KafkaClient(bootstrap_servers)
|
|
40
|
+
if "connection_status" not in st.session_state:
|
|
41
|
+
st.session_state.connection_status = "Connecting"
|
|
42
|
+
if "topics" not in st.session_state:
|
|
43
|
+
st.session_state.topics = None
|
|
44
|
+
try:
|
|
45
|
+
st.session_state.topics = sorted(client.topics())
|
|
46
|
+
st.session_state.connection_status = "Connected"
|
|
47
|
+
except Exception as exc:
|
|
48
|
+
st.session_state.connection_status = "Disconnected"
|
|
49
|
+
st.session_state.connection_error = str(exc)
|
|
50
|
+
|
|
51
|
+
status = st.session_state.connection_status
|
|
52
|
+
st.subheader(f"Status: {status}")
|
|
53
|
+
if status == "Disconnected":
|
|
54
|
+
st.error(f"Kafka connection failed: {st.session_state.get('connection_error', 'unknown error')}")
|
|
55
|
+
if st.button("Test / Refresh Connection"):
|
|
56
|
+
try:
|
|
57
|
+
st.session_state.topics = sorted(client.topics())
|
|
58
|
+
st.session_state.connection_status = "Connected"
|
|
59
|
+
st.session_state.pop("connection_error", None)
|
|
60
|
+
except Exception as exc:
|
|
61
|
+
st.session_state.connection_status = "Disconnected"
|
|
62
|
+
st.session_state.connection_error = str(exc)
|
|
63
|
+
st.rerun()
|
|
64
|
+
if status != "Connected":
|
|
65
|
+
return
|
|
66
|
+
|
|
67
|
+
topics = st.session_state.get("topics") or []
|
|
68
|
+
if st.button("Refresh Topics"):
|
|
69
|
+
try:
|
|
70
|
+
st.session_state.topics = sorted(client.topics())
|
|
71
|
+
except Exception as exc:
|
|
72
|
+
st.error(f"Unable to discover topics: {exc}")
|
|
73
|
+
st.rerun()
|
|
74
|
+
if not topics:
|
|
75
|
+
st.info("No topics are available.")
|
|
76
|
+
return
|
|
77
|
+
|
|
78
|
+
topic = st.selectbox("Topic", topics)
|
|
79
|
+
group_id = st.text_input("Consumer group ID", value=st.session_state.get("group_id", ""))
|
|
80
|
+
if st.button("Generate Temporary Group ID"):
|
|
81
|
+
st.session_state.group_id = generate_group_id()
|
|
82
|
+
st.rerun()
|
|
83
|
+
group_id = st.session_state.get("group_id", group_id)
|
|
84
|
+
modes = {"Latest messages": "latest", "From beginning": "beginning", "From date/time": "from_date", "Date/time range": "range"}
|
|
85
|
+
mode_label = st.radio("Loading mode", list(modes), index=0)
|
|
86
|
+
mode = modes[mode_label]
|
|
87
|
+
start = end = None
|
|
88
|
+
if mode == "latest":
|
|
89
|
+
count = st.number_input("Message count", min_value=1, value=100, step=1)
|
|
90
|
+
elif mode == "beginning":
|
|
91
|
+
count = st.number_input("Maximum message count", min_value=1, value=100, step=1)
|
|
92
|
+
elif mode == "from_date":
|
|
93
|
+
start = _date_time("Start date", "start")
|
|
94
|
+
count = st.number_input("Maximum message count", min_value=1, value=100, step=1)
|
|
95
|
+
else:
|
|
96
|
+
start = _date_time("Start date", "range-start")
|
|
97
|
+
end = _date_time("End date", "range-end")
|
|
98
|
+
count = st.number_input("Maximum message count", min_value=1, value=100, step=1)
|
|
99
|
+
|
|
100
|
+
load_col, clear_col = st.columns(2)
|
|
101
|
+
if load_col.button("Load Messages", type="primary"):
|
|
102
|
+
try:
|
|
103
|
+
validate_count(count)
|
|
104
|
+
st.session_state.messages = client.load_messages(topic, group_id, mode, int(count), start, end)
|
|
105
|
+
except Exception as exc:
|
|
106
|
+
st.error(f"Unable to load messages: {exc}")
|
|
107
|
+
if clear_col.button("Clear Loaded Messages"):
|
|
108
|
+
st.session_state.messages = []
|
|
109
|
+
|
|
110
|
+
messages = st.session_state.get("messages", [])
|
|
111
|
+
if not messages:
|
|
112
|
+
st.info("No messages loaded.")
|
|
113
|
+
return
|
|
114
|
+
st.dataframe(
|
|
115
|
+
[{"Partition": message.partition, "Offset": message.offset, "Timestamp": message.timestamp, "Key": format_key(message.key), "Message preview": format_value(message.value)[:500]} for message in messages],
|
|
116
|
+
use_container_width=True,
|
|
117
|
+
hide_index=True,
|
|
118
|
+
)
|
|
119
|
+
st.subheader("Message details")
|
|
120
|
+
for index, message in enumerate(messages, 1):
|
|
121
|
+
with st.expander(f"#{index} partition={message.partition} offset={message.offset}"):
|
|
122
|
+
st.write({"Partition": message.partition, "Offset": message.offset, "Timestamp": message.timestamp, "Key": format_key(message.key)})
|
|
123
|
+
st.code(format_value(message.value), language="json" if format_value(message.value).lstrip().startswith(("{", "[")) else None)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
if __name__ == "__main__":
|
|
127
|
+
main()
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: kafka-viewer
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A lightweight local Kafka message inspection tool
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Requires-Dist: kafka-python<4,>=3.0.11
|
|
9
|
+
Requires-Dist: streamlit<2,>=1.35
|
|
10
|
+
Provides-Extra: test
|
|
11
|
+
Requires-Dist: pytest<10,>=9.0.3; extra == "test"
|
|
12
|
+
Provides-Extra: build
|
|
13
|
+
Requires-Dist: build<2,>=1; extra == "build"
|
|
14
|
+
Provides-Extra: security
|
|
15
|
+
Requires-Dist: pip-audit<3,>=2.9; extra == "security"
|
|
16
|
+
Dynamic: license-file
|
|
17
|
+
|
|
18
|
+
# kafka-viewer
|
|
19
|
+
|
|
20
|
+
`kafka-viewer` is a lightweight local Streamlit UI for inspecting messages in an unsecured Kafka cluster. Kafka connection details come from a local properties file, and the viewer does not commit consumer offsets.
|
|
21
|
+
|
|
22
|
+
## Installation
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install kafka-viewer
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Configuration
|
|
29
|
+
|
|
30
|
+
Create a local properties file named `kafka-viewer.properties`:
|
|
31
|
+
|
|
32
|
+
```properties
|
|
33
|
+
kafka.bootstrap.servers=localhost:9092
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
The configuration remains local to your machine.
|
|
37
|
+
|
|
38
|
+
## Usage
|
|
39
|
+
|
|
40
|
+
The `--config` option is required:
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
kafka-viewer-unsecured --config /path/to/kafka-viewer.properties
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Windows:
|
|
47
|
+
|
|
48
|
+
```text
|
|
49
|
+
kafka-viewer-unsecured --config C:\path\to\kafka-viewer.properties
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
The UI lets you check Kafka connection status, discover and select topics, enter or generate a temporary consumer group ID, and load:
|
|
53
|
+
|
|
54
|
+
- The latest N messages.
|
|
55
|
+
- Messages from the beginning.
|
|
56
|
+
- Messages from a date/time.
|
|
57
|
+
- Messages within a date/time range.
|
|
58
|
+
|
|
59
|
+
Loaded messages show the partition, offset, timestamp, key, and message value. Messages can be expanded to view the complete message value, including formatted JSON when applicable.
|
|
60
|
+
|
|
61
|
+
## V1 limitations
|
|
62
|
+
|
|
63
|
+
V1 supports unsecured Kafka only. It does not support:
|
|
64
|
+
|
|
65
|
+
- Kafka authentication or security configuration.
|
|
66
|
+
- Avro, Schema Registry, or Protobuf.
|
|
67
|
+
- Publishing messages.
|
|
68
|
+
- Topic or consumer-group administration.
|
|
69
|
+
- Multiple Kafka clusters.
|
|
70
|
+
|
|
71
|
+
## Development
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
python -m pytest
|
|
75
|
+
python -m build
|
|
76
|
+
```
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
kafka_viewer/__init__.py
|
|
5
|
+
kafka_viewer/cli.py
|
|
6
|
+
kafka_viewer/config.py
|
|
7
|
+
kafka_viewer/kafka_client.py
|
|
8
|
+
kafka_viewer/ui.py
|
|
9
|
+
kafka_viewer.egg-info/PKG-INFO
|
|
10
|
+
kafka_viewer.egg-info/SOURCES.txt
|
|
11
|
+
kafka_viewer.egg-info/dependency_links.txt
|
|
12
|
+
kafka_viewer.egg-info/entry_points.txt
|
|
13
|
+
kafka_viewer.egg-info/requires.txt
|
|
14
|
+
kafka_viewer.egg-info/top_level.txt
|
|
15
|
+
tests/test_cli.py
|
|
16
|
+
tests/test_config.py
|
|
17
|
+
tests/test_kafka_client.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
kafka_viewer
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "kafka-viewer"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "A lightweight local Kafka message inspection tool"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"kafka-python>=3.0.11,<4",
|
|
13
|
+
"streamlit>=1.35,<2",
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
[project.optional-dependencies]
|
|
17
|
+
test = ["pytest>=9.0.3,<10"]
|
|
18
|
+
build = ["build>=1,<2"]
|
|
19
|
+
security = ["pip-audit>=2.9,<3"]
|
|
20
|
+
|
|
21
|
+
[project.scripts]
|
|
22
|
+
kafka-viewer-unsecured = "kafka_viewer.cli:main"
|
|
23
|
+
|
|
24
|
+
[tool.setuptools.packages.find]
|
|
25
|
+
include = ["kafka_viewer*"]
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from kafka_viewer import cli
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def test_cli_requires_config_option(monkeypatch, capsys):
|
|
9
|
+
monkeypatch.setattr("sys.argv", ["kafka-viewer-unsecured"])
|
|
10
|
+
|
|
11
|
+
with pytest.raises(SystemExit) as result:
|
|
12
|
+
cli.main()
|
|
13
|
+
|
|
14
|
+
assert result.value.code == 2
|
|
15
|
+
assert "the following arguments are required: --config" in capsys.readouterr().err
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def test_cli_fails_before_starting_ui_for_missing_config_file(tmp_path, monkeypatch):
|
|
19
|
+
monkeypatch.chdir(tmp_path)
|
|
20
|
+
monkeypatch.setattr(cli.subprocess, "call", lambda command: pytest.fail("UI should not start"))
|
|
21
|
+
monkeypatch.setattr("sys.argv", ["kafka-viewer-unsecured", "--config", "missing.properties"])
|
|
22
|
+
|
|
23
|
+
with pytest.raises(FileNotFoundError, match="not found"):
|
|
24
|
+
cli.main()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_cli_passes_config_to_streamlit(tmp_path, monkeypatch):
|
|
28
|
+
config = tmp_path / "viewer.properties"
|
|
29
|
+
config.write_text("kafka.bootstrap.servers=broker:9092\n", encoding="utf-8")
|
|
30
|
+
calls = []
|
|
31
|
+
monkeypatch.setattr(cli.subprocess, "call", lambda command: calls.append(command) or 0)
|
|
32
|
+
monkeypatch.setattr("sys.argv", ["kafka-viewer-unsecured", "--config", str(config)])
|
|
33
|
+
|
|
34
|
+
with pytest.raises(SystemExit) as result:
|
|
35
|
+
cli.main()
|
|
36
|
+
|
|
37
|
+
assert result.value.code == 0
|
|
38
|
+
assert calls and calls[0][-1] == str(config)
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from kafka_viewer.config import ConfigError, load_properties
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def test_load_properties_trims_and_ignores_comments(tmp_path):
|
|
7
|
+
path = tmp_path / "viewer.properties"
|
|
8
|
+
path.write_text("\n # comment\n kafka.bootstrap.servers = broker:9092 \n", encoding="utf-8")
|
|
9
|
+
|
|
10
|
+
assert load_properties(path) == {"kafka.bootstrap.servers": "broker:9092"}
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def test_load_properties_requires_bootstrap_servers(tmp_path):
|
|
14
|
+
path = tmp_path / "viewer.properties"
|
|
15
|
+
path.write_text("other=value\n", encoding="utf-8")
|
|
16
|
+
|
|
17
|
+
with pytest.raises(ConfigError, match="kafka.bootstrap.servers"):
|
|
18
|
+
load_properties(path)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_load_properties_reports_missing_file(tmp_path):
|
|
22
|
+
with pytest.raises(FileNotFoundError, match="not found"):
|
|
23
|
+
load_properties(tmp_path / "missing.properties")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_load_properties_rejects_malformed_line(tmp_path):
|
|
27
|
+
path = tmp_path / "viewer.properties"
|
|
28
|
+
path.write_text("not-a-property\n", encoding="utf-8")
|
|
29
|
+
|
|
30
|
+
with pytest.raises(ConfigError, match="key=value"):
|
|
31
|
+
load_properties(path)
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
from datetime import datetime, timezone
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
from kafka import TopicPartition
|
|
5
|
+
|
|
6
|
+
from kafka_viewer.kafka_client import (
|
|
7
|
+
KafkaClient,
|
|
8
|
+
format_value,
|
|
9
|
+
generate_group_id,
|
|
10
|
+
validate_count,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Record:
|
|
15
|
+
def __init__(self, topic, partition, offset, timestamp, key, value):
|
|
16
|
+
self.topic = topic
|
|
17
|
+
self.partition = partition
|
|
18
|
+
self.offset = offset
|
|
19
|
+
self.timestamp = timestamp
|
|
20
|
+
self.key = key
|
|
21
|
+
self.value = value
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Offset:
|
|
25
|
+
def __init__(self, offset):
|
|
26
|
+
self.offset = offset
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class FakeConsumer:
|
|
30
|
+
records = {
|
|
31
|
+
TopicPartition("events", 0): [Record("events", 0, 0, 1000, None, b'{"a":1}'), Record("events", 0, 1, 3000, b"k", b"last-0")],
|
|
32
|
+
TopicPartition("events", 1): [Record("events", 1, 0, 2000, b"k", b"last-1")],
|
|
33
|
+
}
|
|
34
|
+
instances = []
|
|
35
|
+
|
|
36
|
+
def __init__(self, **kwargs):
|
|
37
|
+
self.kwargs = kwargs
|
|
38
|
+
self.assigned = []
|
|
39
|
+
self.positions = {}
|
|
40
|
+
self.closed = False
|
|
41
|
+
FakeConsumer.instances.append(self)
|
|
42
|
+
|
|
43
|
+
def partitions_for_topic(self, topic):
|
|
44
|
+
return {0, 1}
|
|
45
|
+
|
|
46
|
+
def end_offsets(self, partitions):
|
|
47
|
+
return {partition: len(self.records[partition]) for partition in partitions}
|
|
48
|
+
|
|
49
|
+
def beginning_offsets(self, partitions):
|
|
50
|
+
return {partition: 0 for partition in partitions}
|
|
51
|
+
|
|
52
|
+
def offsets_for_times(self, requested):
|
|
53
|
+
return {partition: Offset(0) for partition in requested}
|
|
54
|
+
|
|
55
|
+
def assign(self, partitions):
|
|
56
|
+
self.assigned = partitions
|
|
57
|
+
|
|
58
|
+
def seek(self, partition, offset):
|
|
59
|
+
self.positions[partition] = offset
|
|
60
|
+
|
|
61
|
+
def poll(self, timeout_ms):
|
|
62
|
+
batch = {}
|
|
63
|
+
for partition in self.assigned:
|
|
64
|
+
position = self.positions[partition]
|
|
65
|
+
records = [record for record in self.records[partition] if record.offset >= position]
|
|
66
|
+
if records:
|
|
67
|
+
record = records[0]
|
|
68
|
+
self.positions[partition] = record.offset + 1
|
|
69
|
+
batch[partition] = [record]
|
|
70
|
+
return batch
|
|
71
|
+
|
|
72
|
+
def position(self, partition):
|
|
73
|
+
return self.positions[partition]
|
|
74
|
+
|
|
75
|
+
def close(self):
|
|
76
|
+
self.closed = True
|
|
77
|
+
|
|
78
|
+
def topics(self):
|
|
79
|
+
return {"events"}
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def setup_function():
|
|
83
|
+
FakeConsumer.instances.clear()
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def test_latest_reads_count_total_across_partitions():
|
|
87
|
+
messages = KafkaClient("broker:9092", FakeConsumer).load_messages("events", "group", "latest", 2)
|
|
88
|
+
|
|
89
|
+
assert [(message.partition, message.offset) for message in messages] == [(1, 0), (0, 1)]
|
|
90
|
+
assert FakeConsumer.instances[-1].kwargs["enable_auto_commit"] is False
|
|
91
|
+
assert FakeConsumer.instances[-1].closed is True
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def test_from_beginning_is_bounded():
|
|
95
|
+
messages = KafkaClient("broker:9092", FakeConsumer).load_messages("events", "group", "beginning", 2)
|
|
96
|
+
|
|
97
|
+
assert len(messages) == 2
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def test_date_range_filters_record_timestamps():
|
|
101
|
+
messages = KafkaClient("broker:9092", FakeConsumer).load_messages(
|
|
102
|
+
"events",
|
|
103
|
+
"group",
|
|
104
|
+
"range",
|
|
105
|
+
10,
|
|
106
|
+
datetime.fromtimestamp(1.5, timezone.utc),
|
|
107
|
+
datetime.fromtimestamp(2.5, timezone.utc),
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
assert [(message.partition, message.offset) for message in messages] == [(1, 0)]
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def test_format_value_json_and_fallback():
|
|
114
|
+
assert format_value(b'{"name":"Ada"}') == '{\n "name": "Ada"\n}'
|
|
115
|
+
assert format_value(b"plain text") == "plain text"
|
|
116
|
+
assert format_value(None) == "<null>"
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def test_group_id_is_recognizable():
|
|
120
|
+
assert generate_group_id().startswith("kafka-viewer-")
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
@pytest.mark.parametrize("value", [0, -1, "nope"])
|
|
124
|
+
def test_validate_count_rejects_invalid_values(value):
|
|
125
|
+
with pytest.raises(ValueError, match="positive integer"):
|
|
126
|
+
validate_count(value)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def test_validate_count_accepts_positive_integer():
|
|
130
|
+
assert validate_count("3") == 3
|