logcatx 0.0.1__py3-none-any.whl

logcatx/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2026 Junbo Zheng
+
+"""logcatx: local web dashboard for offline embedded dual-core log analysis."""
+
+# Single source of truth — pyproject.toml reads this via [tool.hatch.version].
+__version__ = "0.0.1"

logcatx/analyzer.py

@@ -0,0 +1,143 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2026 Junbo Zheng
+
+"""Deterministic statistics over parsed lines and mined templates."""
+
+from collections import Counter, defaultdict
+
+ERROR_LEVELS = {"ERROR", "ALERT", "EMERG", "CRIT"}
+
+
+def overview(lines, unparsed, templates, blank=0):
+    total = len(lines) + len(unparsed) + blank
+    level_counts = Counter(ln.level for ln in lines)
+    module_counts = Counter(f"[{ln.core}] [{ln.module}]" for ln in lines)
+    err_tpls = [
+        t for t in templates if any(lv in ERROR_LEVELS for lv in t.level_counts)
+    ]
+    err_tpls.sort(key=lambda t: -t.count)
+    times = [ln.time for ln in lines]
+    return {
+        "total": total,
+        "parsed": len(lines),
+        "unparsed": len(unparsed),
+        "time_start": min(times).isoformat() if times else None,
+        "time_end": max(times).isoformat() if times else None,
+        "level_counts": dict(level_counts),
+        "module_top": module_counts.most_common(15),
+        "error_top": [
+            {"id": t.id, "text": t.text, "namespace": t.namespace, "count": t.count}
+            for t in err_tpls[:20]
+        ],
+    }
+
+
+def _buckets(lines, bucket_seconds):
+    if not lines:
+        return []
+    start = min(ln.time for ln in lines).timestamp()
+    end = max(ln.time for ln in lines).timestamp()
+    n = int((end - start) // bucket_seconds) + 1
+    return [
+        {"start": start + i * bucket_seconds, "level_counts": {}} for i in range(n)
+    ], start
+
+
+def timeline(lines, bucket_seconds):
+    if not lines:
+        return {"bucket_seconds": bucket_seconds, "buckets": []}
+    buckets, start = _buckets(lines, bucket_seconds)
+    for ln in lines:
+        idx = int((ln.time.timestamp() - start) // bucket_seconds)
+        b = buckets[idx]
+        b["level_counts"][ln.level] = b["level_counts"].get(ln.level, 0) + 1
+    return {"bucket_seconds": bucket_seconds, "buckets": buckets}
+
+
+def template_series(templates, lines, bucket_seconds):
+    """Per-template count per bucket, keyed by template id."""
+    if not lines or not templates:
+        return {}
+    buckets, start = _buckets(lines, bucket_seconds)
+    n = len(buckets)
+    series = {t.id: [0] * n for t in templates}
+    by_tpl = defaultdict(list)
+    for ln in lines:
+        if ln.template_id >= 0:
+            by_tpl[ln.template_id].append(ln)
+    for t in templates:
+        for ln in by_tpl.get(t.id, []):
+            idx = int((ln.time.timestamp() - start) // bucket_seconds)
+            series[t.id][idx] += 1
+    return series
+
+
+def bursts(series, sigma=5, min_count=10):
+    """Templates whose bucket count jumps beyond mean + sigma*std.
+
+    Stats are leave-one-out: the candidate bucket is excluded from its own
+    baseline, otherwise a single spike inflates std enough to hide itself.
+    """
+    hits = []
+    for tid, counts in series.items():
+        if len(counts) < 3:
+            continue
+        total = sum(counts)
+        n = len(counts)
+        for i, c in enumerate(counts):
+            if c < min_count:
+                continue
+            others = total - c
+            mean = others / (n - 1)
+            var = sum((x - mean) ** 2 for j, x in enumerate(counts) if j != i) / (n - 1)
+            std = var**0.5
+            if c > mean + sigma * std:
+                hits.append(
+                    {
+                        "template_id": tid,
+                        "bucket_index": i,
+                        "count": c,
+                        "mean": mean,
+                        "sigma": std,
+                    }
+                )
+    hits.sort(key=lambda h: -h["count"])
+    return hits
+
+
+def cooccur(lines, window=1):
+    """Groups of >=2 distinct ERROR-ish template ids within the same second."""
+    groups = []
+    by_sec = defaultdict(list)
+    for ln in lines:
+        if ln.level in ERROR_LEVELS and ln.template_id >= 0:
+            by_sec[ln.time.replace(microsecond=0)].append(ln)
+    for sec, ls in sorted(by_sec.items()):
+        tids = sorted({ln.template_id for ln in ls})
+        if len(tids) >= 2:
+            groups.append(
+                {
+                    "line_no": ls[0].line_no,
+                    "time": sec.isoformat(),
+                    "template_ids": tids,
+                }
+            )
+    return groups
+
+
+def auto_bucket(lines, max_buckets=1000):
+    if not lines:
+        return 60
+    span = (max(ln.time for ln in lines) - min(ln.time for ln in lines)).total_seconds()
+    if span <= 3600:
+        b = 60
+    elif span <= 6 * 3600:
+        b = 300
+    elif span <= 24 * 3600:
+        b = 600
+    else:
+        b = 3600
+    # clock-reset artifacts can fake a huge span; keep the bucket count sane
+    while span / b > max_buckets:
+        b *= 2
+    return b

logcatx/cli.py

@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2026 Junbo Zheng
+
+"""CLI entrypoint: logcatx <logfile> [--port N] [--no-browser]."""
+
+import argparse
+import logging
+import socket
+import threading
+import webbrowser
+from importlib.metadata import PackageNotFoundError, version
+
+import uvicorn
+
+from .server import create_app, load_session
+
+
+def _version() -> str:
+    # Installed metadata is the source of truth; fall back to the in-tree
+    # __init__ string for zero-install runs (main.py shim, editable checkouts).
+    try:
+        return version("logcatx")
+    except PackageNotFoundError:
+        from . import __version__
+
+        return __version__
+
+
+def _port_free(port):
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        return s.connect_ex(("127.0.0.1", port)) != 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    logging.basicConfig(
+        level=logging.DEBUG,
+        datefmt="%Y-%m-%d %H:%M:%S",
+        format=(
+            "%(asctime)-19s.%(msecs)03d %(levelname)-8s "
+            "%(filename)s %(lineno)-3d %(process)d %(message)s"
+        ),
+    )
+    p = argparse.ArgumentParser(
+        prog="logcatx",
+        description=f"logcatx {_version()} — offline log analysis dashboard.",
+    )
+    p.add_argument(
+        "-V",
+        "--version",
+        action="version",
+        version=f"logcatx {_version()}",
+        help="Show the version and exit",
+    )
+    p.add_argument("logfile")
+    p.add_argument("--port", type=int, default=8000)
+    p.add_argument("--no-browser", action="store_true")
+    args = p.parse_args(argv)
+
+    session = load_session(args.logfile)
+    app = create_app(session)
+
+    port = args.port
+    for _ in range(10):
+        if _port_free(port):
+            break
+        port += 1
+
+    if not args.no_browser:
+        url = f"http://127.0.0.1:{port}/"
+        threading.Timer(1.0, lambda: webbrowser.open(url)).start()
+
+    uvicorn.run(app, host="127.0.0.1", port=port, log_level="warning")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

logcatx/miner.py

@@ -0,0 +1,75 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2026 Junbo Zheng
+
+"""In-house log template miner (Drain-style, Vela-tailored).
+
+Clustering key insight vs stock Drain: the [core] [module] prefix is a
+namespace, never merged across — that is what produced Drain3's catch-all
+blobs on real Vela logs.
+"""
+
+import re
+from dataclasses import dataclass, field
+
+MASKS = [
+    (re.compile(r"\b(?:[0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}\b"), "<MAC>"),
+    (re.compile(r"0x[0-9A-Fa-f]+"), "<HEX>"),
+    (re.compile(r"(?:/[\w.-]+)+"), "<PATH>"),
+    (re.compile(r"\d+\.\d+"), "<NUM>"),
+    (re.compile(r"\d+"), "<NUM>"),
+]
+
+
+@dataclass
+class Template:
+    id: int
+    namespace: str
+    text: str
+    tokens: list = field(default_factory=list)
+    count: int = 0
+    level_counts: dict = field(default_factory=dict)
+    first_line: int = -1
+    last_line: int = -1
+    line_nos: list = field(default_factory=list)
+
+
+def _mask(msg):
+    for rx, tag in MASKS:
+        msg = rx.sub(tag, msg)
+    return msg
+
+
+def mine(lines, sim_th=0.4, max_scan=20):
+    """Cluster masked messages into templates; backfills line.template_id.
+
+    Index key is (namespace, token_count, first token) — the Drain prefix
+    idea flattened — so buckets stay small on varied text (a flat
+    (namespace, len) bucket degrades to O(n^2) at a million lines).
+    """
+    clusters = []
+    index = {}
+    for ln in lines:
+        ns = f"[{ln.core}] [{ln.module}]"
+        toks = _mask(ln.message).split()
+        key = (ns, len(toks), toks[0] if toks else "")
+        bucket = index.setdefault(key, [])
+        hit = None
+        for t in bucket[:max_scan]:
+            same = sum(1 for a, b in zip(t.tokens, toks) if a == b)
+            if same / max(len(toks), 1) >= sim_th:
+                hit = t
+                break
+        if hit is None:
+            hit = Template(len(clusters), ns, " ".join(toks), list(toks))
+            clusters.append(hit)
+            bucket.append(hit)
+        # differing positions generalize to <*>
+        hit.tokens = [a if a == b else "<*>" for a, b in zip(hit.tokens, toks)]
+        hit.text = " ".join(hit.tokens)
+        hit.count += 1
+        hit.level_counts[ln.level] = hit.level_counts.get(ln.level, 0) + 1
+        hit.line_nos.append(ln.line_no)
+        hit.first_line = hit.line_nos[0]
+        hit.last_line = ln.line_no
+        ln.template_id = hit.id
+    return clusters

logcatx/parser.py

@@ -0,0 +1,76 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2026 Junbo Zheng
+
+"""Vela dual-core log line parser.
+
+Line format (fixed for v1):
+    [MM/DD HH:MM:SS] [tid] [LEVEL] [core] [module] message
+The year is not part of the line; it is inferred (see parse_file).
+"""
+
+import re
+from dataclasses import dataclass, field
+from datetime import datetime
+
+LINE_RE = re.compile(
+    r"^\[(\d{2}/\d{2}) (\d{2}:\d{2}:\d{2})\]\s*\[\s*(\d+)\]\s*"
+    r"\[\s*(\w+)\s*\]\s*\[\s*(\w+)\s*\]\s*(?:\[\s*([^\]]+)\s*\]\s*)?(.*)$"
+)
+
+
+@dataclass
+class LogLine:
+    time: datetime
+    tid: int
+    level: str
+    core: str
+    module: str
+    message: str
+    line_no: int
+    template_id: int = -1
+
+
+@dataclass
+class RawLine:
+    text: str
+    line_no: int
+
+
+@dataclass
+class ParseResult:
+    lines: list = field(default_factory=list)
+    unparsed: list = field(default_factory=list)
+    blank: int = 0
+    errors: int = 0
+
+
+def parse_file(path, year=None):
+    """Parse a Vela log file.
+
+    Lines that do not match the format are kept in `unparsed` (never dropped).
+    The year rolls over when month/day goes backwards (Dec 31 -> Jan 01).
+    """
+    year = year or datetime.now().year
+    res = ParseResult()
+    with open(path, encoding="utf-8", errors="replace") as f:
+        prev = None
+        for no, raw in enumerate(f, 1):
+            line = raw.rstrip("\n")
+            m = LINE_RE.match(line)
+            if not m:
+                if line.strip():
+                    res.unparsed.append(RawLine(line, no))
+                else:
+                    res.blank += 1
+                continue
+            mmdd, hms, tid, level, core, module, msg = m.groups()
+            mon, day = map(int, mmdd.split("/"))
+            h, mi, s = map(int, hms.split(":"))
+            # Only a true Dec -> Jan transition rolls the year; a reboot clock
+            # reset (e.g. 09/21 -> 01/01 mid-log) must not.
+            if prev and prev.month == 12 and mon == 1:
+                year += 1
+            t = datetime(year, mon, day, h, mi, s)
+            prev = t
+            res.lines.append(LogLine(t, int(tid), level, core, module or "", msg, no))
+    return res

logcatx/server.py

@@ -0,0 +1,236 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2026 Junbo Zheng
+
+"""FastAPI session server: everything in-memory, no persistence."""
+
+import os
+from dataclasses import dataclass, field
+
+from fastapi import FastAPI
+from fastapi.responses import FileResponse
+from fastapi.staticfiles import StaticFiles
+
+from .analyzer import (
+    ERROR_LEVELS,
+    auto_bucket,
+    bursts,
+    cooccur,
+    overview,
+    template_series,
+    timeline,
+)
+from .miner import mine
+from .parser import parse_file
+
+MAX_PAGE = 1000
+
+
+def _clamp(limit):
+    # non-positive limit means "everything" -> MAX_PAGE; never a negative slice
+    return MAX_PAGE if limit <= 0 else min(limit, MAX_PAGE)
+
+
+@dataclass
+class Session:
+    parse: object = None
+    templates: list = field(default_factory=list)
+    name: str = ""
+    # line_no -> parsed line or raw line, for context expansion
+    by_line: dict = field(default_factory=dict)
+
+
+def load_session(path):
+    res = parse_file(path)
+    tpls = mine(res.lines)
+    s = Session(parse=res, templates=tpls, name=path)
+    for ln in res.lines:
+        s.by_line[ln.line_no] = ln
+    for raw in res.unparsed:
+        s.by_line[raw.line_no] = raw
+    return s
+
+
+def create_app(session: Session) -> FastAPI:
+    app = FastAPI()
+    s = session
+
+    @app.get("/api/overview")
+    def api_overview():
+        return overview(
+            s.parse.lines, s.parse.unparsed, s.templates, blank=s.parse.blank
+        )
+
+    @app.get("/api/trend")
+    def api_trend(template_ids: str = "", bucket: int = 0):
+        """Frequency-over-time series for user-selected templates (any level)."""
+        b = bucket if bucket > 0 else auto_bucket(s.parse.lines)
+        ids = []
+        for part in template_ids.split(","):
+            part = part.strip()
+            if part.isdigit():
+                ids.append(int(part))
+        tpls = [t for t in s.templates if t.id in ids]
+        series = template_series(tpls, s.parse.lines, b)
+        return {
+            "bucket_seconds": b,
+            "series": {str(t.id): series[t.id] for t in tpls},
+            "templates": {str(t.id): t.text for t in tpls},
+        }
+
+    @app.get("/api/templates")
+    def api_templates(
+        search: str = "",
+        sort: str = "count",
+        order: str = "desc",
+        limit: int = 100,
+        offset: int = 0,
+    ):
+        items = list(s.templates)
+        if search:
+            items = [
+                t
+                for t in items
+                if search.lower() in t.text.lower()
+                or search.lower() in t.namespace.lower()
+            ]
+        reverse = order == "desc"
+        keyf = (lambda t: t.text.lower()) if sort == "text" else (lambda t: t.count)
+        items.sort(key=keyf, reverse=reverse)
+        total = len(items)
+        page = items[max(0, offset) : max(0, offset) + _clamp(limit)]
+        return {
+            "total": total,
+            "items": [
+                {
+                    "id": t.id,
+                    "namespace": t.namespace,
+                    "text": t.text,
+                    "count": t.count,
+                    "level_counts": t.level_counts,
+                    "first_line": t.first_line,
+                    "last_line": t.last_line,
+                }
+                for t in page
+            ],
+        }
+
+    @app.get("/api/timeline")
+    def api_timeline(bucket: int = 0):
+        b = bucket if bucket > 0 else auto_bucket(s.parse.lines)
+        tl = timeline(s.parse.lines, b)
+        err_tpls = [
+            t for t in s.templates if any(lv in ERROR_LEVELS for lv in t.level_counts)
+        ]
+        err_tpls.sort(key=lambda t: -t.count)
+        series = template_series(err_tpls[:200], s.parse.lines, b)
+        return {**tl, "per_template": series, "bursts": bursts(series)}
+
+    @app.get("/api/logs")
+    def api_logs(
+        level: str = "",
+        core: str = "",
+        module: str = "",
+        template_id: int = -1,
+        q: str = "",
+        line: int = -1,
+        start: float = -1,
+        end: float = -1,
+        context: int = 0,
+        limit: int = 100,
+        offset: int = 0,
+    ):
+        if line >= 0:
+            lo = max(1, line - context)
+            hi = line + context
+            rows = [s.by_line[i] for i in range(lo, hi + 1) if i in s.by_line]
+            return {
+                "total": len(rows),
+                "items": [
+                    {
+                        "line_no": r.line_no,
+                        "text": _render(r),
+                        "parsed": hasattr(r, "level"),
+                        "context": True,
+                    }
+                    for r in rows
+                ],
+            }
+        items = s.parse.lines
+        if level:
+            items = [ln for ln in items if ln.level == level]
+        if core:
+            items = [ln for ln in items if ln.core == core]
+        if module:
+            items = [ln for ln in items if module.lower() in ln.module.lower()]
+        if template_id >= 0:
+            items = [ln for ln in items if ln.template_id == template_id]
+        if q:
+            items = [ln for ln in items if q.lower() in ln.message.lower()]
+        if start >= 0:
+            items = [ln for ln in items if ln.time.timestamp() >= start]
+        if end >= 0:
+            items = [ln for ln in items if ln.time.timestamp() < end]
+        # unparsed lines ride along unless a parsed-only filter is active;
+        # they are part of the file and must stay browsable
+        unparsed = s.parse.unparsed
+        if q:
+            unparsed = [u for u in unparsed if q.lower() in u.text.lower()]
+        if level or core or module or template_id >= 0 or start >= 0 or end >= 0:
+            unparsed = []
+        merged = sorted(
+            [
+                {
+                    "line_no": ln.line_no,
+                    "time": ln.time.strftime("%m/%d %H:%M:%S"),
+                    "tid": ln.tid,
+                    "level": ln.level,
+                    "core": ln.core,
+                    "module": ln.module,
+                    "message": ln.message,
+                    "template_id": ln.template_id,
+                    "parsed": True,
+                    "context": False,
+                }
+                for ln in items
+            ]
+            + [
+                {
+                    "line_no": u.line_no,
+                    "text": u.text,
+                    "parsed": False,
+                    "context": False,
+                }
+                for u in unparsed
+            ],
+            key=lambda x: x["line_no"],
+        )
+        total = len(merged)
+        page = merged[max(0, offset) : max(0, offset) + _clamp(limit)]
+        return {"total": total, "items": page}
+
+    @app.get("/api/cooccur")
+    def api_cooccur():
+        return {"groups": cooccur(s.parse.lines)}
+
+    app.mount("/static", StaticFiles(directory=_static_dir()), name="static")
+
+    @app.get("/")
+    def index():
+        return FileResponse(os.path.join(_static_dir(), "index.html"))
+
+    return app
+
+
+def _render(r):
+    if hasattr(r, "level"):  # parsed LogLine
+        return (
+            f"[{r.time.strftime('%m/%d %H:%M:%S')}] [{r.tid:2d}] "
+            f"[{r.level:>5}] [{r.core}] [{r.module}] {r.message}"
+        )
+    return r.text
+
+
+def _static_dir():
+    import os
+
+    return os.path.join(os.path.dirname(__file__), "static")