PyPI - wisent-compute - Versions diffs - 0.4.332__tar.gz → 0.4.334__tar.gz - Mend

@@ -44,10 +44,13 @@ def _pypi_latest(pkg: str) -> str | None:
     cached = _CACHE.get(pkg)
     if cached and (time.time() - cached[0]) < _CACHE_TTL_SECONDS:
         return cached[1]
-    with urllib.request.urlopen(
-        f"https://pypi.org/pypi/{pkg}/json",
-    ) as resp:
-        data = json.loads(resp.read())
+    try:
+        with urllib.request.urlopen(
+            f"https://pypi.org/pypi/{pkg}/json", timeout=20,
+        ) as resp:
+            data = json.loads(resp.read())
+    except Exception:
+        return None
     releases = data.get("releases") or {}
     if not releases:
         return None
@@ -82,10 +85,13 @@ def wisent_import_ok() -> tuple[bool, str]:
     claiming jobs that will fail their first `python -m wisent...` line.
     """
     import subprocess, sys
-    res = subprocess.run(
-        [sys.executable, "-c", "import wisent; import wisent_compute"],
-        capture_output=True, text=True,
-    )
+    try:
+        res = subprocess.run(
+            [sys.executable, "-c", "import wisent; import wisent_compute"],
+            capture_output=True, text=True, timeout=60,
+        )
+    except subprocess.TimeoutExpired:
+        return False, "import wisent/wisent_compute timed out after 60s"
     if res.returncode == 0:
         return True, ""
     return False, (res.stderr or res.stdout or "(no output)").strip()[:400]

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wisent-compute
-Version: 0.4.332
+Version: 0.4.334
 Summary: Job queue and compute management for Wisent GPU workloads
 Requires-Python: >=3.10
 License-File: LICENSE

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "wisent-compute"
-version = "0.4.332"
+version = "0.4.334"
 description = "Job queue and compute management for Wisent GPU workloads"
 requires-python = ">=3.10"
 dependencies = [

@@ -115,7 +115,7 @@ done
 # Only computed when disk is tight (>80%); avoids running du on every
 # beacon tick on a healthy host.
 top_consumers='[]'
-if [ "${disk_pct:-0}" -ge 80 ]; then
+if [ "${disk_pct:-0}" -ge 80 ] || [ "${disk_avail_gb:-999999}" -lt 30 ]; then
     top_consumers=$(
         {
             /usr/bin/du -h --max-depth=1 "$HOME_DIR" 2>/dev/null

@@ -10,5 +10,4 @@ Environment=PATH={PATH}
 Environment=GOOGLE_APPLICATION_CREDENTIALS={ADC_PATH}
 Environment=GOOGLE_CLOUD_PROJECT={GCP_PROJECT}
 Environment=WC_HEALTH_UNITS=wisent-agent.service
-ExecStartPre=-/bin/sh -c 'BEACON=$(/usr/bin/python3 -c "import wisent_compute, os; print(os.path.join(os.path.dirname(wisent_compute.__file__), \"deploy\", \"host_health_beacon.sh\"))" 2>/dev/null); [ -n "$BEACON" ] && [ -f "$BEACON" ] && /bin/cp "$BEACON" {HOME}/host_health_beacon.sh || true'
 ExecStart=/bin/bash {HOME}/host_health_beacon.sh

@@ -179,3 +179,33 @@ def setup_agent_staging(log_fn) -> str | None:
         f"{target} (disk-backed, {free_gb:.0f}G free)"
     )
     return target
+def _flush_staging_child(path: str) -> None:
+    from wisent.core.reading.modules.utilities.data.sources.hf.hf_writers import (
+        flush_staging_dir,
+    )
+    flush_staging_dir(path)
+def flush_fleet_staging(path: str, log_fn, timeout_s: int = 300) -> bool:
+    """Flush agent-wide HF staging without wedging the agent loop forever."""
+    if not os.path.isdir(path) or not any(os.scandir(path)):
+        return False
+    import multiprocessing as mp
+    proc = mp.Process(target=_flush_staging_child, args=(path,))
+    proc.start()
+    proc.join(timeout_s)
+    if proc.is_alive():
+        proc.terminate()
+        proc.join(10)
+        log_fn(f"fleet staging flush timed out after {timeout_s}s; keeping staging dir")
+        return False
+    if proc.exitcode != 0:
+        log_fn(f"fleet staging flush failed rc={proc.exitcode}; keeping staging dir")
+        return False
+    shutil.rmtree(path)
+    os.makedirs(path, exist_ok=True)
+    log_fn("flushed fleet staging dir to HF (1 commit)")
+    return True

@@ -30,7 +30,7 @@ from .local.helpers import (
     _staging_size_gb,
     _vast_has_renter,
 )
-from .local.disk.staging import setup_agent_staging
+from .local.disk.staging import flush_fleet_staging, setup_agent_staging
 POLL_INTERVAL = 10
@@ -153,12 +153,7 @@ def run_agent(gpu_type: str = "", idle_shutdown: bool = False, kind: str = "loca
             except Exception:
                 pass
         if time.time() - last_fleet_flush > FLEET_FLUSH_INTERVAL or _staging_size_gb(fleet_staging) > 5:
-            from wisent.core.reading.modules.utilities.data.sources.hf.hf_writers import flush_staging_dir
-            if os.path.isdir(fleet_staging) and any(os.scandir(fleet_staging)):
-                flush_staging_dir(fleet_staging)
-                shutil.rmtree(fleet_staging)
-                os.makedirs(fleet_staging, exist_ok=True)
-                _log("flushed fleet staging dir to HF (1 commit)")
+            flush_fleet_staging(fleet_staging, _log)
             last_fleet_flush = time.time()
         t = lookup_self(hostname, source="auto")
         if t and t.kind == "local":

wisent-compute 0.4.332__tar.gz → 0.4.334__tar.gz