libthx 0.3.1__tar.gz → 0.4.0__tar.gz

{libthx-0.3.1 → libthx-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: libthx
-Version: 0.3.1
+Version: 0.4.0
 Summary: Architecture experimentation and training infrastructure.
 Requires-Python: <3.14,>=3.11
 Description-Content-Type: text/markdown

{libthx-0.3.1 → libthx-0.4.0}/libthx.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: libthx
-Version: 0.3.1
+Version: 0.4.0
 Summary: Architecture experimentation and training infrastructure.
 Requires-Python: <3.14,>=3.11
 Description-Content-Type: text/markdown

{libthx-0.3.1 → libthx-0.4.0}/libthx.egg-info/SOURCES.txt

@@ -12,6 +12,7 @@ tests/test_analysis.py
 tests/test_base.py
 tests/test_bundle.py
 tests/test_checkpoint.py
+tests/test_cleanup.py
 tests/test_cli.py
 tests/test_cli_browser.py
 tests/test_cli_configure.py
@@ -81,6 +82,7 @@ theseus/base/job.py
 theseus/base/topology.py
 theseus/cli/__init__.py
 theseus/cli/app.py
+theseus/cli/cleanup.py
 theseus/cli/configure.py
 theseus/cli/jobs.py
 theseus/cli/run.py

{libthx-0.3.1 → libthx-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "libthx"
-version = "0.3.1"
+version = "0.4.0"
 description = "Architecture experimentation and training infrastructure."
 readme = "README.md"
 requires-python = ">=3.11,<3.14"

libthx-0.4.0/tests/test_cleanup.py

@@ -0,0 +1,145 @@
+"""Value-table maintenance preserves committed rows and reader snapshots."""
+
+import os
+import importlib
+import shutil
+from time import time
+from unittest.mock import Mock
+
+import pyarrow as pa
+import pytest
+from deltalake import DeltaTable, write_deltalake
+from deltalake.exceptions import CommitFailedError
+from deltalake.table import TableOptimizer
+from typer.testing import CliRunner
+
+from theseus.cli.app import app
+from theseus.store import ObjectStore
+import theseus.store as store_module
+
+
+def seed(values):
+    for value in range(3):
+        write_deltalake(values, pa.table({"value": [value]}), mode="append")
+
+
+@pytest.mark.parametrize("expire", [False, True])
+def test_cleanup_compacts_and_vacuums_with_selected_retention(tmp_path, monkeypatch, expire):
+    values = tmp_path / "objects" / "values"
+    seed(values)
+    snapshot = DeltaTable(values)
+    old_files = snapshot.file_uris()
+    expired = values / "orphan.parquet"
+    shutil.copyfile(old_files[0], expired)
+    old_time = time() - 8 * 24 * 3600
+    os.utime(expired, (old_time, old_time))
+    recent = values / "recent-orphan.parquet"
+    shutil.copyfile(old_files[0], recent)
+    cleanup_module = importlib.import_module("theseus.cli.cleanup")
+    delay = Mock()
+    monkeypatch.setattr(cleanup_module, "sleep", delay)
+
+    result = CliRunner().invoke(app, ["cleanup", str(tmp_path), *(["--expire"] if expire else [])])
+
+    assert result.exit_code == 0, result.output
+    assert "Value table cleanup" in result.output
+    assert "CLEANUP | compacting" in result.output
+    assert "CLEANUP | vacuuming" in result.output
+    assert not expired.exists()
+    assert recent.exists() is not expire
+    assert len(DeltaTable(values).file_uris()) == 1
+    if expire:
+        delay.assert_called_once_with(10)
+        assert "ALL ACTIVE WRITERS WILL CRASH IN 10 SECONDS, C-c ONCE TO CANCEL" in result.output
+        assert all(not os.path.exists(path) for path in old_files)
+    else:
+        delay.assert_not_called()
+        # Default retention preserves readers bound before compaction.
+        assert sorted(snapshot.to_pyarrow_table()["value"].to_pylist()) == [0, 1, 2]
+    assert sorted(DeltaTable(values).to_pyarrow_table()["value"].to_pylist()) == [0, 1, 2]
+
+
+def test_cleanup_expire_can_be_cancelled_before_mutation(tmp_path, monkeypatch):
+    cleanup_module = importlib.import_module("theseus.cli.cleanup")
+    delay = Mock(side_effect=KeyboardInterrupt)
+    compact = Mock()
+    vacuum = Mock()
+    monkeypatch.setattr(cleanup_module, "sleep", delay)
+    monkeypatch.setattr(ObjectStore, "compact", compact)
+    monkeypatch.setattr(DeltaTable, "vacuum", vacuum)
+
+    result = CliRunner().invoke(app, ["cleanup", str(tmp_path), "--expire"])
+
+    assert result.exit_code != 0
+    assert "ALL ACTIVE WRITERS WILL CRASH IN 10 SECONDS, C-c ONCE TO CANCEL" in result.output
+    delay.assert_called_once_with(10)
+    compact.assert_not_called()
+    vacuum.assert_not_called()
+
+
+def test_cleanup_missing_table_is_noop(tmp_path):
+    result = CliRunner().invoke(app, ["cleanup", str(tmp_path)])
+    assert result.exit_code == 0, result.output
+    assert "no Delta value table" in result.output
+    assert not (tmp_path / "objects").exists()
+
+
+@pytest.mark.parametrize("schema_change", [False, True])
+def test_compaction_preserves_concurrent_append_and_retries_schema_change(
+    tmp_path, monkeypatch, schema_change,
+):
+    values = tmp_path / "objects" / "values"
+    seed(values)
+    original = TableOptimizer.compact
+    calls = 0
+
+    def racing_compact(optimizer):
+        nonlocal calls
+        calls += 1
+        if calls == 1:
+            data = {"value": [3]}
+            if schema_change:
+                data["new_column"] = [4]
+            write_deltalake(values, pa.table(data), mode="append", schema_mode="merge")
+        return original(optimizer)
+
+    monkeypatch.setattr(TableOptimizer, "compact", racing_compact)
+    monkeypatch.setattr(store_module, "_DELTA_IO_RETRY_SECONDS", 0)
+    ObjectStore.compact(tmp_path)
+
+    assert calls == (2 if schema_change else 1)
+    rows = DeltaTable(values).to_pyarrow_table()
+    assert sorted(rows["value"].to_pylist()) == [0, 1, 2, 3]
+    if schema_change:
+        assert rows.filter(pa.compute.equal(rows["value"], 3))["new_column"].to_pylist() == [4]
+
+
+def test_close_logs_compaction_failure_after_flushing(tmp_path, monkeypatch):
+    from theseus.base import Node
+
+    store = ObjectStore.local(tmp_path)
+    store.value(Node(name="test"), {"value": 1})
+    warning = Mock()
+    monkeypatch.setattr(store_module.logger, "warning", warning)
+    monkeypatch.setattr(
+        ObjectStore, "compact", Mock(side_effect=CommitFailedError("conflict")),
+    )
+    store.close()
+    warning.assert_called_once()
+    assert store.query().select() == [{"value": 1}]
+
+
+def test_cleanup_stops_before_vacuum_after_repeated_conflicts(tmp_path, monkeypatch):
+    seed(tmp_path / "objects" / "values")
+    compact = Mock(side_effect=CommitFailedError("concurrent schema change"))
+    vacuum = Mock()
+    monkeypatch.setattr(TableOptimizer, "compact", compact)
+    monkeypatch.setattr(DeltaTable, "vacuum", vacuum)
+    monkeypatch.setattr(store_module, "_DELTA_IO_RETRY_SECONDS", 0)
+
+    result = CliRunner().invoke(app, ["cleanup", str(tmp_path)])
+
+    assert result.exit_code == 1
+    assert "concurrent schema change" in result.output
+    assert compact.call_count == store_module._DELTA_IO_ATTEMPTS
+    vacuum.assert_not_called()

{libthx-0.3.1 → libthx-0.4.0}/tests/test_execute_bootstrap.py

@@ -603,6 +603,38 @@ def test_bootstrap_autobatch_isolates_probes_and_overrides_real_run(
     assert "probing training.per_device_batch_size=512" in contents
 
 
+def test_bootstrap_autobatch_waits_for_delayed_log_writer(
+    render_bootstrap: Callable[[int], tuple[Path, Path, Path, Path]],
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
+    import shlex
+
+    script, dispatch, calls, logs = render_bootstrap(-1)
+    monkeypatch.setenv("THESEUS_DISPATCH_PROBE_COOLDOWN_SECONDS", "0")
+    real_tee = shutil.which("tee")
+    assert real_tee is not None
+    delayed_tee = tmp_path / "bin" / "tee"
+    delayed_tee.write_text(
+        '#!/usr/bin/env bash\n'
+        'if [[ "$1" == */autobatch-*.log ]]; then sleep 2; fi\n'
+        f'exec {shlex.quote(real_tee)} "$@"\n'
+    )
+    delayed_tee.chmod(0o755)
+
+    result = subprocess.run(
+        ["bash", script, dispatch], text=True, capture_output=True, timeout=30,
+    )
+
+    assert result.returncode == 0, result.stdout + result.stderr
+    output = result.stdout + result.stderr
+    assert "No such file or directory" not in output
+    run_calls = [line for line in calls.read_text().splitlines() if line.startswith("run")]
+    assert len(run_calls) == 3
+    assert "dispatch.json training.per_device_batch_size=512" in run_calls[-1]
+    assert "RESOURCE_EXHAUSTED" in (logs / "project-group-name-abc123.0.log").read_text()
+
+
 def test_bootstrap_autobatch_skips_clamped_duplicate_candidates(
     render_bootstrap: Callable[[int], tuple[Path, Path, Path, Path]],
     monkeypatch: pytest.MonkeyPatch,

{libthx-0.3.1 → libthx-0.4.0}/tests/test_store.py

@@ -203,7 +203,7 @@ def test_value_batch_flush_and_lifecycle(
     assert store.thread is not None and store.thread is not first_thread
     store.value(node, {"restarted": 1})
     store.close()
-    assert len(DeltaTable(store.values()).file_uris()) == 2
+    assert len(DeltaTable(store.values()).file_uris()) == 1
     assert store.query().node(node).select() == [{"loss": 2.0, "restarted": 1}]
 
 
@@ -222,7 +222,7 @@ def test_value_time_flush(
     assert store.query().node(node).select() == [{"value": 1}]
 
 
-def test_close_does_not_rewrite_or_remove_committed_parts(
+def test_close_compacts_without_vacuuming_committed_parts(
     hardware: HardwareResult, monkeypatch: pytest.MonkeyPatch
 ) -> None:
     monkeypatch.setattr(store_module, "_VALUE_BATCH_SIZE", 1)
@@ -235,8 +235,11 @@ def test_close_does_not_rewrite_or_remove_committed_parts(
     before = {part: part.read_bytes() for part in parts}
     version = DeltaTable(store.values()).version()
     store.close()
+    compacted = DeltaTable(store.values())
+    assert len(compacted.file_uris()) == 1
+    assert compacted.version() > version
     store.close()
-    assert DeltaTable(store.values()).version() == version
+    assert DeltaTable(store.values()).version() == compacted.version()
     assert {part: part.read_bytes() for part in parts} == before
     assert store.query().all() == [
         Node(name="train", nonce="abcdef", seq=0),
@@ -304,7 +307,7 @@ def test_parallel_processes_publish_without_collisions(
 
     second.close()
     parts = DeltaTable(first.values()).file_uris()
-    assert len(parts) == 2
+    assert len(parts) == 1
     assert {row["_x_prc_idx"] for row in read_rows(first)} == {0, 1}
 
     process_index = 0

{libthx-0.3.1 → libthx-0.4.0}/theseus/cli/app.py

@@ -14,6 +14,7 @@ from theseus.cli.configure import configure
 from theseus.cli.submit import submit
 from theseus.cli.run import run
 from theseus.cli.jobs import jobs
+from theseus.cli.cleanup import cleanup
 
 
 app = typer.Typer(name="theseus", no_args_is_help=True)
@@ -22,6 +23,7 @@ app.command()(configure)
 app.command()(submit)
 app.command()(run)
 app.command()(jobs)
+app.command()(cleanup)
 
 
 @app.callback()
@@ -42,7 +44,7 @@ def main(
     """Run and inspect Theseus workloads."""
     logger.remove()
     logger.add(
-        lambda message: sys.stderr.write(message),
+        sys.stderr,
         level="DEBUG" if verbose else "INFO",
         format="<green>{time:HH:mm:ss}</green> | <level>{level: <8}</level> | <level>{message}</level>",
         backtrace=False,

libthx-0.4.0/theseus/cli/cleanup.py

@@ -0,0 +1,58 @@
+"""Compact and vacuum a Theseus root's Delta value table."""
+
+from pathlib import Path
+from time import sleep
+
+from deltalake import DeltaTable
+from loguru import logger
+from rich.console import Console
+from rich.table import Table
+import typer
+
+from theseus.store import ObjectStore
+
+
+def cleanup(
+    root: Path = typer.Argument(..., exists=True, file_okay=False),
+    expire: bool = typer.Option(
+        False,
+        "--expire",
+        help="Immediately delete obsolete files. Can break active readers and writers; waits 10 seconds before starting.",
+    ),
+) -> None:
+    """Compact ROOT/objects/values and vacuum; --expire bypasses retention."""
+    values = root / "objects" / "values"
+    if expire:
+        logger.warning(
+            "ALL ACTIVE WRITERS WILL CRASH IN 10 SECONDS, C-c ONCE TO CANCEL"
+        )
+        logger.info(
+            "--expire deletes obsolete and uncommitted files without retention; "
+            "active readers and writers may fail."
+        )
+        sleep(10)
+    try:
+        logger.info("CLEANUP | compacting {}", values)
+        metrics = ObjectStore.compact(root)
+        if metrics is None:
+            logger.info("CLEANUP | no Delta value table at {}", values)
+            return
+        logger.info(
+            "CLEANUP | vacuuming {} with {}",
+            values,
+            "zero retention" if expire else "table retention",
+        )
+        removed = DeltaTable(values).vacuum(
+            dry_run=False,
+            full=True,
+            retention_hours=0 if expire else None,
+            enforce_retention_duration=not expire,
+        )
+        summary = Table("Statistic", "Files", title="Value table cleanup")
+        summary.add_row("Compacted inputs", str(metrics["numFilesRemoved"]))
+        summary.add_row("Compacted outputs", str(metrics["numFilesAdded"]))
+        summary.add_row("Expired files deleted", str(len(removed)))
+        Console().print(summary)
+    except Exception as error:
+        logger.error("CLEANUP | {}", error)
+        raise typer.Exit(1) from error

{libthx-0.3.1 → libthx-0.4.0}/theseus/execute/bootstrap.sh

@@ -552,6 +552,13 @@ if grep -Eq \
             "${NONCE}-${probe_id}-autobatch-${candidate}"
 
         echo "[bootstrap] probing training.per_device_batch_size=$candidate"
+        # Create the buffer before polling, and keep the writer PID so a
+        # fast-failing probe cannot be classified before its output drains.
+        : > "$probe_log"
+        probe_pipe="$BOOTSTRAP_STAGING/autobatch-${candidate}.fifo"
+        mkfifo "$probe_pipe"
+        tee "$probe_log" < "$probe_pipe" &
+        probe_log_sink_pid=$!
         set +e
         # The probe tee keeps a classification buffer; its stdout still
         # flows through the outer tee into the durable cluster log.
@@ -560,7 +567,7 @@ if grep -Eq \
             uv run --no-sync python -m theseus.execute.run "$probe_spec" \
                 "training.per_device_batch_size=$candidate" \
                 "${probe_overrides[@]}" \
-            > >(tee "$probe_log") 2>&1 &
+            > "$probe_pipe" 2>&1 &
         MAIN_CHILD_PID=$!
         probe_completed=0
         while kill -0 "$MAIN_CHILD_PID" 2>/dev/null; do
@@ -574,8 +581,15 @@ if grep -Eq \
         wait "$MAIN_CHILD_PID"
         probe_exit_code=$?
         MAIN_CHILD_PID=""
+        wait "$probe_log_sink_pid"
+        probe_log_exit_code=$?
         set -e
+        rm -f "$probe_pipe"
 
+        if [[ "$probe_log_exit_code" -ne 0 ]]; then
+            echo "[bootstrap] ERROR: autobatch log sink failed with exit $probe_log_exit_code"
+            exit "$probe_log_exit_code"
+        fi
         if [[ "$probe_exit_code" -eq 0 || "$probe_completed" -eq 1 ]]; then
             batch_size_override="training.per_device_batch_size=$candidate"
             break

{libthx-0.3.1 → libthx-0.4.0}/theseus/store.py

@@ -25,6 +25,7 @@ from typing import (
     Mapping,
     Self,
     Sequence,
+    TypeVar,
     cast,
     overload,
 )
@@ -37,6 +38,7 @@ import pyarrow as pa
 from deltalake import DeltaTable, write_deltalake
 from deltalake.exceptions import CommitFailedError, TableNotFoundError
 from flax import serialization
+from loguru import logger
 
 from theseus.base import JobSpec, Node, PyTree
 from theseus.base.hardware import HardwareResult, local as local_hardware
@@ -46,6 +48,7 @@ _VALUE_BATCH_SIZE = 4096
 _VALUE_FLUSH_SECONDS = 30.0
 _DELTA_IO_ATTEMPTS = 3
 _DELTA_IO_RETRY_SECONDS = 0.25
+_IOResult = TypeVar("_IOResult")
 _chdb_query = cast(Callable[..., pa.Table], chdb.query)
 _METADATA_TYPES: dict[str, pa.DataType] = {
     "_x_blob": pa.string(),
@@ -153,7 +156,7 @@ class ObjectReader:
         )
 
     @staticmethod
-    def _retry_io(operation: Callable[[], Any]) -> Any:
+    def _retry_io(operation: Callable[[], _IOResult]) -> _IOResult:
         for attempt in range(_DELTA_IO_ATTEMPTS):
             try:
                 return operation()
@@ -248,10 +251,34 @@ class ObjectStore(ObjectReader):
         """
         return cls(local_hardware(str(root_dir), "-"))
 
+    @classmethod
+    def compact(cls, root_dir: str | Path) -> dict[str, Any] | None:
+        """Compact ROOT/objects/values without starting a writer or vacuuming.
+
+        Reload the table after concurrent commit conflicts. An absent table
+        is a no-op; obsolete files remain available to existing readers.
+        """
+        values = Path(root_dir) / "objects" / "values"
+        for attempt in range(_DELTA_IO_ATTEMPTS):
+            try:
+                return cast(
+                    dict[str, Any],
+                    cls._retry_io(lambda: DeltaTable(values).optimize.compact()),
+                )
+            except TableNotFoundError:
+                return None
+            except CommitFailedError:
+                if attempt + 1 == _DELTA_IO_ATTEMPTS:
+                    raise
+                sleep(_DELTA_IO_RETRY_SECONDS * 2**attempt)
+        raise AssertionError("unreachable")
+
     def close(self) -> None:
-        """Commit queued values and stop the writer without rewriting files.
+        """Commit queued values, stop the writer, and compact the value table.
 
         A successfully closed store may be restarted with :meth:`start`.
+        Compaction failures are logged without failing committed writes.
+        Closing never vacuums files needed by existing readers.
 
         Raises:
             RuntimeError: If the background writer failed while producing a
@@ -265,6 +292,12 @@ class ObjectStore(ObjectReader):
         self.thread = None
 
         self._raise_writer_error()
+        try:
+            self.compact(self.root().parent)
+        except Exception as error:
+            logger.warning(
+                "STORE | value compaction failed for {}: {}", self.values(), error
+            )
 
     #### public writes ####