tb-like 0.1.0__tar.gz

tb_like-0.1.0/.gitignore

@@ -0,0 +1,17 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+
+# tb_like data / generated artifacts (never commit these)
+data/
+cache/
+runs/
+*.log
+.tblike_cache/

tb_like-0.1.0/.python-version

@@ -0,0 +1 @@
+3.12

tb_like-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Misha Kindulov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tb_like-0.1.0/PKG-INFO

@@ -0,0 +1,121 @@
+Metadata-Version: 2.4
+Name: tb-like
+Version: 0.1.0
+Summary: A faster, columnar TensorBoard-style scalar viewer for many series and many runs.
+Project-URL: Homepage, https://github.com/b0nce/tb_like
+Project-URL: Repository, https://github.com/b0nce/tb_like
+Project-URL: Issues, https://github.com/b0nce/tb_like/issues
+Author-email: Misha Kindulov <kindulov.ml@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: machine-learning,metrics,parquet,plotly,tensorboard,visualization
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: FastAPI
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Visualization
+Requires-Python: >=3.12
+Requires-Dist: fastapi>=0.137.1
+Requires-Dist: joblib>=1.5.3
+Requires-Dist: numpy>=1.26
+Requires-Dist: plotly>=6.8.0
+Requires-Dist: polars>=1.41.2
+Requires-Dist: pyarrow>=24.0.0
+Requires-Dist: tensorboard>=2.20.0
+Requires-Dist: tqdm>=4.68.2
+Requires-Dist: uvicorn[standard]>=0.49.0
+Description-Content-Type: text/markdown
+
+# tb_like
+
+A faster, **columnar** TensorBoard-style scalar viewer — built for **many series and many runs**.
+
+TensorBoard re-parses event files on demand and gets slow when a run has tens of
+thousands of scalar series across hundreds of experiments. `tb_like` instead
+converts TensorBoard event files into per-run **Parquet** once, then serves
+downsampled series on demand to a fast Plotly dashboard. New events are picked
+up incrementally in the background.
+
+## Why it's fast
+
+- **Convert once, read many.** Each run's `events.out.tfevents.*` are parsed into
+  a columnar Parquet file sorted by `(tag, step)`, with row-group statistics so a
+  query for a few tags only touches the matching row groups — even when a run has
+  ~18k series.
+- **Incremental & idempotent.** Ingestion tracks each event file's size and record
+  count, so re-scans only parse new data. A background watcher keeps the cache in
+  sync; parsing is parallelized across event files with `joblib`.
+- **Lazy, prioritized rendering.** The dashboard renders charts only as they
+  scroll into view, fetched through a priority queue (visible first, then
+  neighbors, biased toward the scroll direction).
+- **LTTB downsampling** keeps long curves cheap to draw without losing their shape.
+
+## Install
+
+```bash
+pip install tb-like
+# or
+uv tool install tb-like
+```
+
+## Quick start
+
+A "run" is a directory containing `events.out.tfevents.*` files. Point `tb_like`
+at a directory of runs and open the dashboard — that's it:
+
+```
+my_runs/
+  run_a/  events.out.tfevents.*  config.yaml
+  run_b/  events.out.tfevents.*
+  ...
+```
+
+```bash
+tblike my_runs --port 8000 --jobs 8
+# open http://127.0.0.1:8000
+```
+
+The background watcher discovers runs under the folder, converts any that
+changed to Parquet (parsing event files across `--jobs` worker processes), keeps
+the cache in sync, and serves them — no separate build step. The cache lives in
+`<runs_dir>/.tblike_cache` by default (override with `--cache-dir`).
+
+## Dashboard features
+
+- Hierarchical, searchable **tag tree** (regex filter) with smart grouping:
+  path compression of `a.b.c` chains, numeric-enumeration collapsing
+  (`…expert_idx_∗`), and layer indices kept as their own levels.
+- Multi-run overlay, unified hover, EMA smoothing, log-y, step vs. relative-time
+  x-axis, and **outlier clipping** by value percentiles.
+- Collapsible per-group chart sections, resizable sidebar, and a one-click
+  **Refresh selected** that re-ingests from disk and rebuilds the plots.
+
+## CLI
+
+```
+tblike <runs_dir> [--port P] [--host H] [--cache-dir D] [-j JOBS] [--no-watch]
+                                            # the main command: serve + auto-ingest
+```
+
+Advanced / scriptable subcommands:
+
+```
+tblike convert RUN_DIR [RUN_ID] [-j JOBS]   # ingest one run into Parquet (one-off)
+tblike scan                                 # one incremental ingest pass, no server
+```
+
+## How it stores data
+
+```
+cache/<run_id>/
+    data/seg-00000.parquet   # one immutable segment per ingest pass
+    index.json               # tags, per-file ingest state, metadata
+    meta.json                # tiny summary used for fast run listing
+```
+
+Reads union all segments and de-duplicate `(tag, step)` by latest `wall_time`.
+
+## License
+
+MIT — see [LICENSE](LICENSE).

tb_like-0.1.0/README.md

@@ -0,0 +1,92 @@
+# tb_like
+
+A faster, **columnar** TensorBoard-style scalar viewer — built for **many series and many runs**.
+
+TensorBoard re-parses event files on demand and gets slow when a run has tens of
+thousands of scalar series across hundreds of experiments. `tb_like` instead
+converts TensorBoard event files into per-run **Parquet** once, then serves
+downsampled series on demand to a fast Plotly dashboard. New events are picked
+up incrementally in the background.
+
+## Why it's fast
+
+- **Convert once, read many.** Each run's `events.out.tfevents.*` are parsed into
+  a columnar Parquet file sorted by `(tag, step)`, with row-group statistics so a
+  query for a few tags only touches the matching row groups — even when a run has
+  ~18k series.
+- **Incremental & idempotent.** Ingestion tracks each event file's size and record
+  count, so re-scans only parse new data. A background watcher keeps the cache in
+  sync; parsing is parallelized across event files with `joblib`.
+- **Lazy, prioritized rendering.** The dashboard renders charts only as they
+  scroll into view, fetched through a priority queue (visible first, then
+  neighbors, biased toward the scroll direction).
+- **LTTB downsampling** keeps long curves cheap to draw without losing their shape.
+
+## Install
+
+```bash
+pip install tb-like
+# or
+uv tool install tb-like
+```
+
+## Quick start
+
+A "run" is a directory containing `events.out.tfevents.*` files. Point `tb_like`
+at a directory of runs and open the dashboard — that's it:
+
+```
+my_runs/
+  run_a/  events.out.tfevents.*  config.yaml
+  run_b/  events.out.tfevents.*
+  ...
+```
+
+```bash
+tblike my_runs --port 8000 --jobs 8
+# open http://127.0.0.1:8000
+```
+
+The background watcher discovers runs under the folder, converts any that
+changed to Parquet (parsing event files across `--jobs` worker processes), keeps
+the cache in sync, and serves them — no separate build step. The cache lives in
+`<runs_dir>/.tblike_cache` by default (override with `--cache-dir`).
+
+## Dashboard features
+
+- Hierarchical, searchable **tag tree** (regex filter) with smart grouping:
+  path compression of `a.b.c` chains, numeric-enumeration collapsing
+  (`…expert_idx_∗`), and layer indices kept as their own levels.
+- Multi-run overlay, unified hover, EMA smoothing, log-y, step vs. relative-time
+  x-axis, and **outlier clipping** by value percentiles.
+- Collapsible per-group chart sections, resizable sidebar, and a one-click
+  **Refresh selected** that re-ingests from disk and rebuilds the plots.
+
+## CLI
+
+```
+tblike <runs_dir> [--port P] [--host H] [--cache-dir D] [-j JOBS] [--no-watch]
+                                            # the main command: serve + auto-ingest
+```
+
+Advanced / scriptable subcommands:
+
+```
+tblike convert RUN_DIR [RUN_ID] [-j JOBS]   # ingest one run into Parquet (one-off)
+tblike scan                                 # one incremental ingest pass, no server
+```
+
+## How it stores data
+
+```
+cache/<run_id>/
+    data/seg-00000.parquet   # one immutable segment per ingest pass
+    index.json               # tags, per-file ingest state, metadata
+    meta.json                # tiny summary used for fast run listing
+```
+
+Reads union all segments and de-duplicate `(tag, step)` by latest `wall_time`.
+
+## License
+
+MIT — see [LICENSE](LICENSE).

tb_like-0.1.0/pyproject.toml

@@ -0,0 +1,44 @@
+[project]
+name = "tb-like"
+version = "0.1.0"
+description = "A faster, columnar TensorBoard-style scalar viewer for many series and many runs."
+readme = "README.md"
+requires-python = ">=3.12"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Misha Kindulov", email = "kindulov.ml@gmail.com" }]
+keywords = ["tensorboard", "parquet", "plotly", "machine-learning", "visualization", "metrics"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3",
+    "Topic :: Scientific/Engineering :: Visualization",
+    "Framework :: FastAPI",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "fastapi>=0.137.1",
+    "joblib>=1.5.3",
+    "numpy>=1.26",
+    "plotly>=6.8.0",
+    "polars>=1.41.2",
+    "pyarrow>=24.0.0",
+    "tensorboard>=2.20.0",
+    "tqdm>=4.68.2",
+    "uvicorn[standard]>=0.49.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/b0nce/tb_like"
+Repository = "https://github.com/b0nce/tb_like"
+Issues = "https://github.com/b0nce/tb_like/issues"
+
+[project.scripts]
+tblike = "tblike.cli:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["tblike"]

tb_like-0.1.0/tblike/__init__.py

@@ -0,0 +1,8 @@
+"""tb_like: a faster, columnar TensorBoard-style scalar viewer.
+
+Pipeline: TensorBoard event files -> per-run columnar Parquet -> on-demand,
+downsampled reads served to a Plotly dashboard. New events are picked up
+incrementally in a background watcher.
+"""
+
+__version__ = "0.1.0"

tb_like-0.1.0/tblike/cli.py

@@ -0,0 +1,201 @@
+"""Command line entrypoints for tb_like.
+
+Primary usage — just point it at a folder of runs and open the dashboard:
+
+    tblike <runs_dir> [--port 8000] [--jobs 8]
+
+The background watcher converts runs to Parquet automatically and keeps the
+cache in sync. Advanced/scriptable subcommands are also available:
+
+    tblike convert <run_dir> [run_id]   # ingest one run -> parquet (one-off)
+    tblike scan                         # one incremental ingest pass, no server
+    tblike build-runs --count 200       # make symlinked test runs (dev)
+    tblike clone <src_run_id> --count N # fan a converted run out (dev)
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import sys
+import time
+
+from .convert import convert_run, load_index, meta_from_index
+from .watcher import Watcher, discover_runs
+
+
+def cmd_build_runs(args: argparse.Namespace) -> None:
+    src = os.path.abspath(args.source)
+    os.makedirs(args.runs_dir, exist_ok=True)
+    for i in range(args.count):
+        link = os.path.join(args.runs_dir, f"{args.prefix}{i:03d}")
+        if os.path.islink(link) or os.path.exists(link):
+            os.remove(link) if os.path.islink(link) else None
+        if not os.path.exists(link):
+            os.symlink(src, link)
+    print(f"created {args.count} symlinked runs in {args.runs_dir}/ -> {src}")
+
+
+def _file_progress(prefix: str):
+    """Return an on_file(done, total, name) callback backed by a tqdm bar."""
+    from tqdm import tqdm
+
+    bar = {"t": None}
+
+    def cb(done: int, total: int, name: str) -> None:
+        if bar["t"] is None:
+            bar["t"] = tqdm(total=total, desc=prefix, unit="file", dynamic_ncols=True)
+        bar["t"].n = done
+        bar["t"].refresh()
+        if done >= total:
+            bar["t"].close()
+
+    return cb
+
+
+def cmd_convert(args: argparse.Namespace) -> None:
+    run_id = args.run_id or os.path.basename(os.path.normpath(args.run_dir))
+    cache_run_dir = os.path.join(args.cache_dir, run_id)
+    t0 = time.time()
+    res = convert_run(
+        args.run_dir, cache_run_dir, run_id,
+        on_file=_file_progress(run_id), n_jobs=args.jobs,
+    )
+    dt = time.time() - t0
+    print(
+        f"[{run_id}] +{res.new_rows:,} rows (total {res.total_rows:,}), "
+        f"{res.num_tags} tags, segment={res.segment} in {dt:.1f}s ({args.jobs} jobs)"
+    )
+
+
+def cmd_clone(args: argparse.Namespace) -> None:
+    """Fan a single converted run out into N runs by hardlinking its Parquet.
+
+    Used only for the scale test: the 200 runs are symlinks to identical bytes,
+    so their columnar output is identical too. Hardlinks keep disk ~flat while
+    giving each run its own independent index.json / display name.
+    """
+    src_dir = os.path.join(args.cache_dir, args.src_run_id)
+    src_index = load_index(src_dir)
+    if not src_index:
+        print(f"source run {args.src_run_id!r} not converted yet", file=sys.stderr)
+        sys.exit(1)
+    src_data = os.path.join(src_dir, "data")
+    segments = src_index.get("segments", [])
+
+    for i in range(args.count):
+        rid = f"{args.prefix}{i:03d}"
+        dst = os.path.join(args.cache_dir, rid)
+        dst_data = os.path.join(dst, "data")
+        os.makedirs(dst_data, exist_ok=True)
+        for seg in segments:
+            link = os.path.join(dst_data, seg)
+            if os.path.exists(link):
+                os.remove(link)
+            try:
+                os.link(os.path.join(src_data, seg), link)  # hardlink, ~free
+            except OSError:
+                shutil.copy2(os.path.join(src_data, seg), link)
+        idx = dict(src_index)
+        idx["run_id"] = rid
+        base = src_index.get("display_name", args.src_run_id)
+        idx["display_name"] = f"{base} #{i:03d}"
+        idx["source_dir"] = os.path.abspath(os.path.join(args.runs_dir, rid))
+        with open(os.path.join(dst, "index.json"), "w") as fh:
+            json.dump(idx, fh)
+        with open(os.path.join(dst, "meta.json"), "w") as fh:
+            json.dump(meta_from_index(idx), fh)
+    print(f"cloned {args.src_run_id} -> {args.count} runs ({len(segments)} segments each)")
+
+
+def cmd_scan(args: argparse.Namespace) -> None:
+    w = Watcher(args.runs_dir, args.cache_dir)
+    runs = discover_runs(args.runs_dir)
+    print(f"discovered {len(runs)} runs; scanning...")
+    t0 = time.time()
+    res = w.scan_once()
+    print(f"scan done in {time.time()-t0:.1f}s: {res}")
+
+
+DEFAULT_JOBS = max(1, (os.cpu_count() or 2) - 1)
+
+
+def run_serve(argv: list[str]) -> None:
+    import uvicorn
+
+    p = argparse.ArgumentParser(
+        prog="tblike",
+        description="Serve the tb_like dashboard for a folder of runs (auto-ingests in the background).",
+    )
+    p.add_argument("runs_dir", help="folder containing run subdirs with events.out.tfevents.* files")
+    p.add_argument("--port", type=int, default=8000)
+    p.add_argument("--host", default="127.0.0.1")
+    p.add_argument("--cache-dir", default=None,
+                   help="Parquet cache directory (default: <runs_dir>/.tblike_cache)")
+    p.add_argument("-j", "--jobs", type=int, default=DEFAULT_JOBS,
+                   help=f"parallel parse workers for background conversion (default: {DEFAULT_JOBS})")
+    p.add_argument("--interval", type=float, default=10.0, help="watcher poll interval, seconds")
+    p.add_argument("--no-watch", action="store_true", help="serve only; do not ingest in the background")
+    args = p.parse_args(argv)
+
+    if not os.path.isdir(args.runs_dir):
+        p.error(f"runs_dir not found: {args.runs_dir}")
+    cache = args.cache_dir or os.path.join(args.runs_dir, ".tblike_cache")
+    os.environ["TBLIKE_RUNS"] = os.path.abspath(args.runs_dir)
+    os.environ["TBLIKE_CACHE"] = os.path.abspath(cache)
+    os.environ["TBLIKE_WATCH"] = "0" if args.no_watch else "1"
+    os.environ["TBLIKE_INTERVAL"] = str(args.interval)
+    os.environ["TBLIKE_JOBS"] = str(args.jobs)
+    print(f"tb_like → http://{args.host}:{args.port}   runs={args.runs_dir}  cache={cache}")
+    uvicorn.run("tblike.server:app", host=args.host, port=args.port, reload=False)
+
+
+def run_advanced(argv: list[str]) -> None:
+    p = argparse.ArgumentParser(prog="tblike")
+    p.add_argument("--runs-dir", default="runs")
+    p.add_argument("--cache-dir", default="cache")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    c = sub.add_parser("convert", help="ingest one run into parquet")
+    c.add_argument("run_dir")
+    c.add_argument("run_id", nargs="?")
+    c.add_argument("-j", "--jobs", type=int, default=DEFAULT_JOBS,
+                   help="parallel worker processes for event-file parsing")
+    c.set_defaults(func=cmd_convert)
+
+    s = sub.add_parser("scan", help="one incremental ingest pass, no server")
+    s.set_defaults(func=cmd_scan)
+
+    b = sub.add_parser("build-runs", help="create symlinked test runs (dev)")
+    b.add_argument("--source", default="data")
+    b.add_argument("--count", type=int, default=200)
+    b.add_argument("--prefix", default="run_")
+    b.set_defaults(func=cmd_build_runs)
+
+    cl = sub.add_parser("clone", help="fan a converted run out into N runs (dev)")
+    cl.add_argument("src_run_id")
+    cl.add_argument("--count", type=int, default=200)
+    cl.add_argument("--prefix", default="run_")
+    cl.set_defaults(func=cmd_clone)
+
+    args = p.parse_args(argv)
+    args.func(args)
+
+
+ADVANCED = {"convert", "scan", "build-runs", "clone"}
+
+
+def main(argv: list[str] | None = None) -> None:
+    argv = list(sys.argv[1:] if argv is None else argv)
+    if argv and argv[0] in ADVANCED:
+        run_advanced(argv)            # power-user subcommands
+    elif not argv or argv[0] in ("-h", "--help"):
+        run_serve(["--help"])         # default command's help
+    else:
+        run_serve(argv)               # `tblike <runs_dir> [opts]`
+
+
+if __name__ == "__main__":
+    main()