tinytrainlog 0.1.1__tar.gz → 0.1.2__tar.gz

{tinytrainlog-0.1.1 → tinytrainlog-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tinytrainlog
-Version: 0.1.1
+Version: 0.1.2
 Summary: Tiny train logger
 Keywords: pytorch,metrics,logging,training,machine-learning
 Author: jdh
@@ -10,4 +10,9 @@ Requires-Python: >=3.12
 Project-URL: Homepage, https://github.com/jdhouseholder/tinytrainlog
 Description-Content-Type: text/markdown
 
-# 🚅🚅🚅 Tiny Train Log  🚅🚅🚅
+# 🚅🪵 Tiny Train Log  🚅🪵
+
+The *structured log* for multi-server research scale ml project metrics.
+Stores runs in a queryable and mergable database for easy multi-server data collection!
+
+## Finally SQL based *post-hoc* analysis for my research projects!

tinytrainlog-0.1.2/README.md

@@ -0,0 +1,6 @@
+# 🚅🪵 Tiny Train Log  🚅🪵
+
+The *structured log* for multi-server research scale ml project metrics.
+Stores runs in a queryable and mergable database for easy multi-server data collection!
+
+## Finally SQL based *post-hoc* analysis for my research projects!

{tinytrainlog-0.1.1 → tinytrainlog-0.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "tinytrainlog"
-version = "0.1.1"
+version = "0.1.2"
 description = "Tiny train logger"
 readme = "README.md"
 authors = [

{tinytrainlog-0.1.1 → tinytrainlog-0.1.2}/src/tinytrainlog/_names.py

@@ -1,5 +1,5 @@
 import random
-from pathlib import Path
+import sqlite3
 
 ADJECTIVES = [
     "bold",
@@ -108,8 +108,8 @@ NOUNS = [
 ]
 
 
-def generate_run_name(root_dir: Path, max_attempts: int = 100) -> str:
-    existing = {p.name for p in root_dir.iterdir()} if root_dir.exists() else set()
+def generate_run_name(conn: sqlite3.Connection, max_attempts: int = 100) -> str:
+    existing = {row[0] for row in conn.execute("SELECT name FROM runs").fetchall()}
     for _ in range(max_attempts):
         name = f"{random.choice(ADJECTIVES)}-{random.choice(NOUNS)}"
         if name not in existing:

tinytrainlog-0.1.2/src/tinytrainlog/metrics_logger.py

@@ -0,0 +1,234 @@
+import json
+import shutil
+import socket
+import sqlite3
+from pathlib import Path
+
+from ._names import generate_run_name
+
+_SCHEMA = """\
+CREATE TABLE IF NOT EXISTS runs (
+    name       TEXT PRIMARY KEY,
+    machine_id TEXT,
+    created_at TEXT NOT NULL DEFAULT (datetime('now'))
+);
+CREATE TABLE IF NOT EXISTS config (
+    run_name   TEXT NOT NULL REFERENCES runs(name),
+    key        TEXT NOT NULL,
+    value      TEXT NOT NULL,
+    updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+    UNIQUE(run_name, key)
+);
+CREATE TABLE IF NOT EXISTS tags (
+    run_name   TEXT NOT NULL REFERENCES runs(name),
+    tag        TEXT NOT NULL,
+    created_at TEXT NOT NULL DEFAULT (datetime('now')),
+    UNIQUE(run_name, tag)
+);
+CREATE TABLE IF NOT EXISTS steps (
+    run_name   TEXT    NOT NULL REFERENCES runs(name),
+    step       INTEGER NOT NULL,
+    key        TEXT    NOT NULL,
+    value      REAL    NOT NULL,
+    created_at TEXT    NOT NULL DEFAULT (datetime('now'))
+);
+CREATE TABLE IF NOT EXISTS epochs (
+    run_name   TEXT    NOT NULL REFERENCES runs(name),
+    epoch      INTEGER NOT NULL,
+    key        TEXT    NOT NULL,
+    value      REAL    NOT NULL,
+    created_at TEXT    NOT NULL DEFAULT (datetime('now'))
+);
+CREATE TABLE IF NOT EXISTS eval (
+    run_name   TEXT    NOT NULL REFERENCES runs(name),
+    step       INTEGER,
+    epoch      INTEGER,
+    key        TEXT    NOT NULL,
+    value      REAL    NOT NULL,
+    created_at TEXT    NOT NULL DEFAULT (datetime('now'))
+);
+CREATE TABLE IF NOT EXISTS test (
+    run_name   TEXT NOT NULL REFERENCES runs(name),
+    key        TEXT NOT NULL,
+    value      REAL NOT NULL,
+    updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+    UNIQUE(run_name, key)
+);
+"""
+
+_DATA_TABLES = ("config", "tags", "steps", "epochs", "eval", "test")
+
+
+class MetricsLogger:
+    def __init__(
+        self,
+        root_dir: str | Path,
+        run_name: str | None = None,
+        machine_id: str | None = None,
+    ):
+        self.root_dir = Path(root_dir)
+        self.root_dir.mkdir(parents=True, exist_ok=True)
+        self.machine_id = machine_id or socket.gethostname()
+
+        self._db_path = self.root_dir / "runs.db"
+        self._conn = sqlite3.connect(self._db_path)
+        self._conn.executescript(_SCHEMA)
+
+        if run_name is None:
+            run_name = generate_run_name(self._conn)
+        self.run_name = run_name
+
+        self._conn.execute(
+            "INSERT INTO runs (name, machine_id) VALUES (?, ?)"
+            " ON CONFLICT(name) DO UPDATE SET machine_id = excluded.machine_id",
+            (self.run_name, self.machine_id),
+        )
+        self._conn.commit()
+
+        self._checkpoint_dir = self.root_dir / self.run_name / "checkpoints"
+        self._checkpoint_dir.mkdir(parents=True, exist_ok=True)
+
+    @property
+    def run_dir(self) -> Path:
+        return self.root_dir / self.run_name
+
+    def set_config(self, config: dict) -> None:
+        self._conn.executemany(
+            "INSERT INTO config (run_name, key, value) VALUES (?, ?, ?)"
+            " ON CONFLICT(run_name, key) DO UPDATE SET value = excluded.value,"
+            " updated_at = datetime('now')",
+            [(self.run_name, k, json.dumps(v)) for k, v in config.items()],
+        )
+        self._conn.commit()
+
+    def add_tags(self, tags: list[str]) -> None:
+        self._conn.executemany(
+            "INSERT OR IGNORE INTO tags (run_name, tag) VALUES (?, ?)",
+            [(self.run_name, tag) for tag in tags],
+        )
+        self._conn.commit()
+
+    def log_step(self, step: int, **metrics) -> None:
+        self._conn.executemany(
+            "INSERT INTO steps (run_name, step, key, value) VALUES (?, ?, ?, ?)",
+            [(self.run_name, step, k, v) for k, v in metrics.items()],
+        )
+        self._conn.commit()
+
+    def log_epoch(self, epoch: int, **metrics) -> None:
+        self._conn.executemany(
+            "INSERT INTO epochs (run_name, epoch, key, value) VALUES (?, ?, ?, ?)",
+            [(self.run_name, epoch, k, v) for k, v in metrics.items()],
+        )
+        self._conn.commit()
+
+    def log_eval(
+        self, *, step: int | None = None, epoch: int | None = None, **metrics
+    ) -> None:
+        if step is None and epoch is None:
+            raise ValueError("At least one of 'step' or 'epoch' must be provided.")
+        self._conn.executemany(
+            "INSERT INTO eval (run_name, step, epoch, key, value) VALUES (?, ?, ?, ?, ?)",
+            [(self.run_name, step, epoch, k, v) for k, v in metrics.items()],
+        )
+        self._conn.commit()
+
+    def log_test(self, **metrics) -> None:
+        self._conn.executemany(
+            "INSERT INTO test (run_name, key, value) VALUES (?, ?, ?)"
+            " ON CONFLICT(run_name, key) DO UPDATE SET value = excluded.value,"
+            " updated_at = datetime('now')",
+            [(self.run_name, k, v) for k, v in metrics.items()],
+        )
+        self._conn.commit()
+
+    def checkpoint_path(
+        self, step: int | None = None, epoch: int | None = None
+    ) -> Path:
+        if (step is None) == (epoch is None):
+            raise ValueError("Exactly one of 'step' or 'epoch' must be provided.")
+        if step is not None:
+            return self._checkpoint_dir / f"step_{step}.pt"
+        return self._checkpoint_dir / f"epoch_{epoch}.pt"
+
+    @property
+    def checkpoint_dir(self) -> Path:
+        return self._checkpoint_dir
+
+    def delete_run(self, run_name: str) -> None:
+        self._conn.execute("BEGIN")
+        try:
+            for table in _DATA_TABLES:
+                self._conn.execute(
+                    f"DELETE FROM {table} WHERE run_name = ?", (run_name,)
+                )
+            self._conn.execute("DELETE FROM runs WHERE name = ?", (run_name,))
+            self._conn.execute("COMMIT")
+        except Exception:
+            self._conn.execute("ROLLBACK")
+            raise
+
+        run_dir = self.root_dir / run_name
+        if run_dir.exists():
+            shutil.rmtree(run_dir)
+
+    @staticmethod
+    def merge(target_dir: str | Path, source_dir: str | Path) -> None:
+        target_dir = Path(target_dir)
+        source_dir = Path(source_dir)
+        target_db = target_dir / "runs.db"
+        source_db = source_dir / "runs.db"
+
+        if not source_db.exists():
+            raise FileNotFoundError(f"No runs.db found in {source_dir}")
+
+        target_dir.mkdir(parents=True, exist_ok=True)
+        conn = sqlite3.connect(target_db)
+        conn.executescript(_SCHEMA)
+        conn.execute("ATTACH DATABASE ? AS other", (str(source_db),))
+
+        # Check for name conflicts
+        conflicts = conn.execute(
+            "SELECT o.name, o.machine_id, m.machine_id"
+            " FROM other.runs o INNER JOIN main.runs m ON o.name = m.name"
+        ).fetchall()
+        if conflicts:
+            conn.execute("DETACH DATABASE other")
+            conn.close()
+            details = ", ".join(
+                f"'{name}' (source: {src or '?'}, target: {tgt or '?'})"
+                for name, src, tgt in conflicts
+            )
+            raise ValueError(
+                f"Run name conflicts: {details}. "
+                f"Rename the conflicting runs before merging."
+            )
+
+        try:
+            conn.execute("BEGIN")
+            conn.execute("INSERT INTO main.runs SELECT * FROM other.runs")
+            for table in _DATA_TABLES:
+                conn.execute(f"INSERT INTO main.{table} SELECT * FROM other.{table}")
+            conn.execute("COMMIT")
+        except Exception:
+            conn.execute("ROLLBACK")
+            raise
+        finally:
+            conn.execute("DETACH DATABASE other")
+            conn.close()
+
+        # Copy checkpoint directories
+        for run_dir in source_dir.iterdir():
+            if run_dir.is_dir():
+                target_run_dir = target_dir / run_dir.name
+                if not target_run_dir.exists():
+                    shutil.copytree(run_dir, target_run_dir)
+
+    def close(self) -> None:
+        self._conn.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        self.close()

tinytrainlog-0.1.1/README.md

@@ -1 +0,0 @@
-# 🚅🚅🚅 Tiny Train Log  🚅🚅🚅

tinytrainlog-0.1.1/src/tinytrainlog/metrics_logger.py

@@ -1,53 +0,0 @@
-import json
-from pathlib import Path
-
-from ._names import generate_run_name
-
-
-class MetricsLogger:
-    def __init__(self, root_dir: str | Path, run_name: str | None = None):
-        self.root_dir = Path(root_dir)
-        if run_name is None:
-            run_name = generate_run_name(self.root_dir)
-        self.run_name = run_name
-        self.run_dir = self.root_dir / self.run_name
-        self.run_dir.mkdir(parents=True, exist_ok=True)
-        self._checkpoint_dir = self.run_dir / "checkpoints"
-        self._checkpoint_dir.mkdir(exist_ok=True)
-
-    def set_config(self, config: dict) -> None:
-        (self.run_dir / "config.json").write_text(json.dumps(config, indent=2) + "\n")
-
-    def add_tags(self, tags: list[str]) -> None:
-        tags_path = self.run_dir / "tags.json"
-        if tags_path.exists():
-            existing = json.loads(tags_path.read_text())
-        else:
-            existing = []
-        seen = set(existing)
-        for tag in tags:
-            if tag not in seen:
-                existing.append(tag)
-                seen.add(tag)
-        tags_path.write_text(json.dumps(existing, indent=2) + "\n")
-
-    def log_step(self, step: int, **metrics) -> None:
-        with open(self.run_dir / "steps.jsonl", "a") as f:
-            f.write(json.dumps({"step": step, **metrics}) + "\n")
-
-    def log_epoch(self, epoch: int, **metrics) -> None:
-        with open(self.run_dir / "epochs.jsonl", "a") as f:
-            f.write(json.dumps({"epoch": epoch, **metrics}) + "\n")
-
-    def checkpoint_path(
-        self, step: int | None = None, epoch: int | None = None
-    ) -> Path:
-        if (step is None) == (epoch is None):
-            raise ValueError("Exactly one of 'step' or 'epoch' must be provided.")
-        if step is not None:
-            return self._checkpoint_dir / f"step_{step}.pt"
-        return self._checkpoint_dir / f"epoch_{epoch}.pt"
-
-    @property
-    def checkpoint_dir(self) -> Path:
-        return self._checkpoint_dir