hf-xet 1.5.0.dev3__tar.gz → 1.5.1.dev0__tar.gz

{hf_xet-1.5.0.dev3 → hf_xet-1.5.1.dev0}/Cargo.toml

@@ -12,7 +12,13 @@ members = [
     "git_xet",
     "simulation",
 ]
-exclude = ["simulation/chunk_cache_bench", "hf_xet", "wasm/hf_xet_wasm", "wasm/hf_xet_thin_wasm"]
+exclude = [
+    "simulation/chunk_cache_bench",
+    "hf_xet",
+    "wasm/hf_xet_wasm",
+    "wasm/hf_xet_thin_wasm",
+    "examples/xet_pkg_napi",
+]
 
 [workspace.package]
 version = "1.5.2"
@@ -52,7 +58,7 @@ console-subscriber = "0.5"
 countio = { version = "0.3", features = ["futures"] }
 crc32fast = "1.5"
 csv = "1"
-ctor = "0.6"
+ctor = "1"
 dirs = "6.0"
 futures = "0.3"
 humantime = "2.1"

{hf_xet-1.5.0.dev3 → hf_xet-1.5.1.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hf-xet
-Version: 1.5.0.dev3
+Version: 1.5.1.dev0
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Rust

{hf_xet-1.5.0.dev3 → hf_xet-1.5.1.dev0}/hf_xet/Cargo.lock

@@ -633,20 +633,14 @@ dependencies = [
 
 [[package]]
 name = "ctor"
-version = "0.6.3"
+version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "424e0138278faeb2b401f174ad17e715c829512d74f3d1e81eb43365c2e0590e"
+checksum = "5c24d2b2b7c12a2fffb7c5c8fd0dcda7ca14b4600fa2d3701b6079aefb6fa180"
 dependencies = [
- "ctor-proc-macro",
- "dtor",
+ "link-section",
+ "linktime-proc-macro",
 ]
 
-[[package]]
-name = "ctor-proc-macro"
-version = "0.0.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1"
-
 [[package]]
 name = "debugid"
 version = "0.8.0"
@@ -707,21 +701,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "dtor"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "404d02eeb088a82cfd873006cb713fe411306c7d182c344905e101fb1167d301"
-dependencies = [
- "dtor-proc-macro",
-]
-
-[[package]]
-name = "dtor-proc-macro"
-version = "0.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5"
-
 [[package]]
 name = "dunce"
 version = "1.0.5"
@@ -1110,7 +1089,7 @@ dependencies = [
 
 [[package]]
 name = "hf_xet"
-version = "1.5.0-dev3"
+version = "1.5.1-dev0"
 dependencies = [
  "async-trait",
  "hf-xet",
@@ -1617,6 +1596,18 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "link-section"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4641b91711debb59c61b07eb5e30521ed6d9e2bdd9fd04f934e7da3a5bc386d4"
+
+[[package]]
+name = "linktime-proc-macro"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a44cd706ff0d503ee32b2071166510ca27e281228de10cd3aa8d35ff94560f81"
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.12.1"
@@ -1875,15 +1866,14 @@ checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107"
 
 [[package]]
 name = "openssl"
-version = "0.10.76"
+version = "0.10.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf"
+checksum = "bf0b434746ee2832f4f0baf10137e1cabb18cbe6912c69e2e33263c45250f542"
 dependencies = [
  "bitflags 2.11.0",
  "cfg-if 1.0.4",
  "foreign-types",
  "libc",
- "once_cell",
  "openssl-macros",
  "openssl-sys",
 ]
@@ -1916,9 +1906,9 @@ dependencies = [
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.112"
+version = "0.9.115"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb"
+checksum = "158fe5b292746440aa6e7a7e690e55aeb72d41505e2804c23c6973ad0e9c9781"
 dependencies = [
  "cc",
  "libc",
@@ -2553,9 +2543,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
 
 [[package]]
 name = "rustls-webpki"
-version = "0.103.10"
+version = "0.103.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef"
+checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
 dependencies = [
  "aws-lc-rs",
  "ring",

{hf_xet-1.5.0.dev3 → hf_xet-1.5.1.dev0}/hf_xet/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hf_xet"
-version = "1.5.0-dev3"
+version = "1.5.1-dev0"
 edition = "2024"
 license = "Apache-2.0"
 readme = "README.md"

hf_xet-1.5.1.dev0/hf_xet/src/background_progress.rs

@@ -0,0 +1,168 @@
+//! Background progress callbacks for upload commits and download groups.
+//!
+//! A dedicated thread polls Rust-side progress atomics and invokes a Python
+//! callable via [`Python::try_attach`].  The poller sleeps on a condvar so
+//! [`BackgroundProgress::stop_and_join`] can wake it immediately instead of
+//! waiting for the full poll interval.
+
+use std::collections::HashMap;
+use std::sync::{Arc, Condvar, Mutex};
+use std::thread::JoinHandle;
+use std::time::Duration;
+
+use pyo3::exceptions::PyRuntimeError;
+use pyo3::prelude::*;
+use xet_pkg::xet_session::{GroupProgressReport, ItemProgressReport, UniqueId};
+
+fn lock_poisoned_err() -> PyErr {
+    PyRuntimeError::new_err("progress poller lock poisoned")
+}
+
+fn poller_panicked_err() -> PyErr {
+    PyRuntimeError::new_err("progress poller thread panicked")
+}
+
+/// Background thread that polls progress and invokes a Python callback.
+pub(crate) struct BackgroundProgress {
+    /// ``false`` while the poller is running; set to ``true`` in ``stop_and_join``.
+    /// Paired with the condvar so setting shutdown wakes a thread blocked in
+    /// ``wait_timeout`` (the mutex is released for the duration of the wait).
+    wake: Arc<(Mutex<bool>, Condvar)>,
+    join: Mutex<Option<JoinHandle<PyResult<()>>>>,
+    callback: Py<PyAny>,
+}
+
+impl BackgroundProgress {
+    /// Spawn a thread that calls ``snapshot`` every ``interval_ms`` until shutdown,
+    /// the snapshot reports a terminal task state, or the callback fails.
+    pub(crate) fn spawn<F>(py: Python<'_>, callback: Py<PyAny>, interval_ms: u64, mut snapshot: F) -> Self
+    where
+        F: FnMut() -> (GroupProgressReport, HashMap<UniqueId, ItemProgressReport>, bool) + Send + 'static,
+    {
+        let wake = Arc::new((Mutex::new(false), Condvar::new()));
+        let wake_for_thread = Arc::clone(&wake);
+        let callback_for_thread = callback.clone_ref(py);
+        let interval = Duration::from_millis(interval_ms);
+
+        let join = std::thread::spawn(move || {
+            progress_thread_loop(callback_for_thread, interval, &wake_for_thread, &mut snapshot)
+        });
+
+        Self {
+            wake,
+            join: Mutex::new(Some(join)),
+            callback,
+        }
+    }
+
+    /// Request shutdown, wake the poller if it is sleeping, and block until the thread exits.
+    pub(crate) fn stop_and_join(&self) -> PyResult<()> {
+        {
+            let mut guard = self.wake.0.lock().map_err(|_| lock_poisoned_err())?;
+            *guard = true;
+        }
+        self.wake.1.notify_one();
+        let handle = self.join.lock().map_err(|_| lock_poisoned_err())?.take();
+        if let Some(handle) = handle {
+            match handle.join() {
+                Ok(result) => result,
+                Err(_) => Err(poller_panicked_err()),
+            }
+        } else {
+            Ok(())
+        }
+    }
+
+    /// Stop the poller, then invoke the callback once on the calling thread (final snapshot).
+    pub(crate) fn stop_and_emit<F>(&self, py: Python<'_>, snapshot: F) -> PyResult<()>
+    where
+        F: FnOnce() -> (GroupProgressReport, HashMap<UniqueId, ItemProgressReport>),
+    {
+        self.stop_and_join()?;
+        let (group_report, item_reports) = snapshot();
+        self.callback.call1(py, (group_report, item_reports))?;
+        Ok(())
+    }
+}
+
+fn progress_thread_loop<F>(
+    callback: Py<PyAny>,
+    interval: Duration,
+    wake: &Arc<(Mutex<bool>, Condvar)>,
+    snapshot: &mut F,
+) -> PyResult<()>
+where
+    F: FnMut() -> (GroupProgressReport, HashMap<UniqueId, ItemProgressReport>, bool),
+{
+    loop {
+        // Sleep up to `interval`, but return promptly when stop_and_join sets shutdown and notifies.
+        let shutdown = {
+            let guard = wake.0.lock().map_err(|_| lock_poisoned_err())?;
+            let (guard, _) = wake.1.wait_timeout(guard, interval).map_err(|_| lock_poisoned_err())?;
+            *guard
+        };
+
+        if shutdown {
+            break;
+        }
+
+        // Mutex is not held across snapshot or the Python callback.
+        let (group_report, item_reports, is_terminal) = snapshot();
+
+        match Python::try_attach(|py| callback.call1(py, (group_report, item_reports))) {
+            None => break, // interpreter shutting down
+            Some(Ok(_)) => {},
+            Some(Err(e)) => {
+                let _ = Python::try_attach(|py| e.print(py));
+                break;
+            },
+        }
+
+        if is_terminal {
+            break;
+        }
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+    use std::time::{Duration, Instant};
+
+    use pyo3::Python;
+    use pyo3::ffi::c_str;
+    use xet_pkg::xet_session::GroupProgressReport;
+
+    use super::*;
+
+    #[test]
+    fn stop_and_join_returns_before_poll_interval_elapses() {
+        /// Poll interval long enough that a blocking sleep would fail the test.
+        const LONG_INTERVAL_MS: u64 = 60_000;
+        const STOP_DEADLINE: Duration = Duration::from_secs(2);
+
+        Python::attach(|py| -> PyResult<()> {
+            py.run(c_str!("def _bg_progress_test_cb(_group, _items): pass"), None, None)?;
+            let callback: Py<PyAny> = py.eval(c_str!("_bg_progress_test_cb"), None, None)?.unbind();
+
+            let progress = BackgroundProgress::spawn(py, callback, LONG_INTERVAL_MS, || {
+                (GroupProgressReport::default(), HashMap::new(), false)
+            });
+
+            // Let the poller enter its timed wait on the condvar.
+            std::thread::sleep(Duration::from_millis(50));
+
+            let start = Instant::now();
+            progress.stop_and_join()?;
+            let elapsed = start.elapsed();
+
+            assert!(
+                elapsed < STOP_DEADLINE,
+                "stop_and_join took {elapsed:?}; expected prompt return (poll interval is {LONG_INTERVAL_MS}ms)"
+            );
+            Ok(())
+        })
+        .unwrap();
+    }
+}

{hf_xet-1.5.0.dev3 → hf_xet-1.5.1.dev0}/hf_xet/src/lib.rs

@@ -1,3 +1,4 @@
+mod background_progress;
 pub mod config;
 mod headers;
 mod legacy;

{hf_xet-1.5.0.dev3 → hf_xet-1.5.1.dev0}/hf_xet/src/py_file_download_group.rs

@@ -1,6 +1,5 @@
 use std::collections::HashMap;
 use std::sync::{Arc, RwLock};
-use std::time::Duration;
 
 use pyo3::prelude::*;
 use xet_pkg::xet_session::{
@@ -8,11 +7,22 @@ use xet_pkg::xet_session::{
     XetFileInfo, XetSession, XetTaskState,
 };
 
+use crate::background_progress::BackgroundProgress;
 use crate::headers::{build_header_map, build_headers_with_user_agent};
 use crate::py_file_download_handle::PyXetFileDownload;
 use crate::utils::{progress_display, task_state_display, task_state_to_pystate};
 use crate::{PyXetTaskState, blocking_call_with_signal_check, convert_xet_error};
 
+#[inline]
+fn item_reports_from_download_handles(
+    handles: &Arc<RwLock<Vec<XetFileDownload>>>,
+) -> HashMap<UniqueId, ItemProgressReport> {
+    handles
+        .read()
+        .map(|g| g.iter().filter_map(|h| h.progress().map(|p| (h.task_id(), p))).collect())
+        .unwrap_or_default()
+}
+
 // ── build_file_download_group ─────────────────────────────────────────────────
 
 /// Create an :class:`XetFileDownloadGroup` from a session and optional configuration.
@@ -51,38 +61,24 @@ pub(crate) fn build_file_download_group(
             .map_err(convert_xet_error)
     })?;
 
-    let download_handles = if let Some(callback) = progress_callback {
+    let (download_handles, progress) = if let Some(callback) = progress_callback {
         let handles: Arc<RwLock<Vec<XetFileDownload>>> = Arc::new(RwLock::new(Vec::new()));
         let inner = group.clone();
         let handles_for_thread = Arc::clone(&handles);
-        let interval = Duration::from_millis(progress_interval_ms);
-        std::thread::spawn(move || {
-            loop {
-                std::thread::sleep(interval);
-                let is_terminal = !matches!(inner.status(), Ok(XetTaskState::Running) | Ok(XetTaskState::Finalizing));
-                let group_report = inner.progress();
-                let item_reports: HashMap<UniqueId, ItemProgressReport> = handles_for_thread
-                    .read()
-                    .map(|g| g.iter().filter_map(|h| h.progress().map(|p| (h.task_id(), p))).collect())
-                    .unwrap_or_default();
-                let result = Python::attach(|py| callback.call1(py, (group_report, item_reports)));
-                if let Err(e) = result {
-                    Python::attach(|py| e.print(py));
-                    break;
-                }
-                if is_terminal {
-                    break;
-                }
-            }
+        let progress = BackgroundProgress::spawn(py, callback, progress_interval_ms, move || {
+            let is_terminal = !matches!(inner.status(), Ok(XetTaskState::Running) | Ok(XetTaskState::Finalizing));
+            let item_reports = item_reports_from_download_handles(&handles_for_thread);
+            (inner.progress(), item_reports, is_terminal)
         });
-        Some(handles)
+        (Some(handles), Some(progress))
     } else {
-        None
+        (None, None)
     };
 
     Ok(PyXetFileDownloadGroup {
         inner: group,
         download_handles,
+        progress,
     })
 }
 
@@ -103,6 +99,9 @@ pub struct PyXetFileDownloadGroup {
     pub(crate) inner: XetFileDownloadGroup,
     /// Per-file handles shared with the progress thread; None when no callback was registered.
     download_handles: Option<Arc<RwLock<Vec<XetFileDownload>>>>,
+    /// Background thread that polls progress and invokes the Python callback; None when no
+    /// callback was registered. Stopped and joined from ``wait_to_finish`` and ``abort``.
+    progress: Option<BackgroundProgress>,
 }
 
 #[pymethods]
@@ -144,7 +143,7 @@ impl PyXetFileDownloadGroup {
             // Normal exit: wait for all downloads (signal-interruptible).
             self.wait_to_finish(py)?;
         } else {
-            if let Err(e) = self.inner.abort() {
+            if let Err(e) = self.abort() {
                 tracing::warn!("abort() failed during __exit__ exception path: {e}");
             }
         }
@@ -188,13 +187,27 @@ impl PyXetFileDownloadGroup {
     ///
     /// Releases the GIL while waiting, polling for ``KeyboardInterrupt`` every
     /// 100 ms so that Ctrl-C is delivered promptly.
+    ///
+    /// When a progress callback was registered, it is invoked once more on the
+    /// calling thread with the final snapshot after all work completes.
     pub fn wait_to_finish(&self, py: Python<'_>) -> PyResult<XetDownloadGroupReport> {
         let group = self.inner.clone();
-        blocking_call_with_signal_check(py, move || group.finish_blocking())
+        let result = blocking_call_with_signal_check(py, move || group.finish_blocking());
+        if let (Some(handles), Some(progress)) = (&self.download_handles, &self.progress) {
+            // ignore any error from progress update
+            let _ = progress.stop_and_emit(py, || {
+                let item_reports = item_reports_from_download_handles(handles);
+                (self.inner.progress(), item_reports)
+            });
+        }
+        result
     }
 
     /// Cancel all active downloads in this group.
     pub fn abort(&self) -> PyResult<()> {
+        if let Some(progress) = &self.progress {
+            progress.stop_and_join()?;
+        }
         self.inner.abort().map_err(convert_xet_error)
     }
 
@@ -236,6 +249,7 @@ mod tests {
                 .build_blocking()
                 .unwrap(),
             download_handles: None,
+            progress: None,
         };
 
         Python::attach(|py| {
@@ -257,6 +271,7 @@ mod tests {
                 .build_blocking()
                 .unwrap(),
             download_handles: None,
+            progress: None,
         };
 
         Python::attach(|py| {

{hf_xet-1.5.0.dev3 → hf_xet-1.5.1.dev0}/hf_xet/src/py_upload_commit.rs

@@ -1,6 +1,5 @@
 use std::collections::HashMap;
 use std::sync::{Arc, RwLock};
-use std::time::Duration;
 
 use pyo3::prelude::*;
 use xet_pkg::xet_session::{
@@ -62,12 +61,23 @@ fn parse_sha256(py: Python<'_>, sha256: Option<Py<PyAny>>) -> PyResult<Sha256Pol
     }
 }
 
+use crate::background_progress::BackgroundProgress;
 use crate::headers::{build_header_map, build_headers_with_user_agent};
 use crate::py_file_upload_handle::PyXetFileUpload;
 use crate::py_stream_upload_handle::PyXetStreamUpload;
 use crate::utils::{progress_display, task_state_display, task_state_to_pystate};
 use crate::{PyXetTaskState, blocking_call_with_signal_check, convert_xet_error};
 
+#[inline]
+fn item_reports_from_upload_handles(
+    handles: &Arc<RwLock<Vec<XetFileUpload>>>,
+) -> HashMap<UniqueId, ItemProgressReport> {
+    handles
+        .read()
+        .map(|g| g.iter().filter_map(|h| h.progress().map(|p| (h.task_id(), p))).collect())
+        .unwrap_or_default()
+}
+
 // ── build_upload_commit ───────────────────────────────────────────────────────
 
 /// Create an :class:`XetUploadCommit` from a session and optional configuration.
@@ -106,38 +116,24 @@ pub(crate) fn build_upload_commit(
             .map_err(convert_xet_error)
     })?;
 
-    let upload_handles = if let Some(callback) = progress_callback {
+    let (upload_handles, background_progress) = if let Some(callback) = progress_callback {
         let handles: Arc<RwLock<Vec<XetFileUpload>>> = Arc::new(RwLock::new(Vec::new()));
         let inner = commit.clone();
         let handles_for_thread = handles.clone();
-        let interval = Duration::from_millis(progress_interval_ms);
-        std::thread::spawn(move || {
-            loop {
-                std::thread::sleep(interval);
-                let is_terminal = !matches!(inner.status(), Ok(XetTaskState::Running) | Ok(XetTaskState::Finalizing));
-                let group_report = inner.progress();
-                let item_reports: HashMap<UniqueId, ItemProgressReport> = handles_for_thread
-                    .read()
-                    .map(|g| g.iter().filter_map(|h| h.progress().map(|p| (h.task_id(), p))).collect())
-                    .unwrap_or_default();
-                let result = Python::attach(|py| callback.call1(py, (group_report, item_reports)));
-                if let Err(e) = result {
-                    Python::attach(|py| e.print(py));
-                    break;
-                }
-                if is_terminal {
-                    break;
-                }
-            }
+        let background_progress = BackgroundProgress::spawn(py, callback, progress_interval_ms, move || {
+            let is_terminal = !matches!(inner.status(), Ok(XetTaskState::Running) | Ok(XetTaskState::Finalizing));
+            let item_reports = item_reports_from_upload_handles(&handles_for_thread);
+            (inner.progress(), item_reports, is_terminal)
         });
-        Some(handles)
+        (Some(handles), Some(background_progress))
     } else {
-        None
+        (None, None)
     };
 
     Ok(PyXetUploadCommit {
         inner: commit,
         upload_handles,
+        background_progress,
     })
 }
 
@@ -158,6 +154,9 @@ pub struct PyXetUploadCommit {
     pub(crate) inner: XetUploadCommit,
     /// Per-file handles shared with the progress thread; None when no callback was registered.
     upload_handles: Option<Arc<RwLock<Vec<XetFileUpload>>>>,
+    /// Background thread that polls progress and invokes the Python callback; None when no
+    /// callback was registered. Stopped and joined from ``wait_to_finish`` and ``abort``.
+    background_progress: Option<BackgroundProgress>,
 }
 
 #[pymethods]
@@ -205,7 +204,7 @@ impl PyXetUploadCommit {
             self.wait_to_finish(py)?;
         } else {
             // Exception: cancel uploads.
-            if let Err(e) = self.inner.abort() {
+            if let Err(e) = self.abort() {
                 tracing::warn!("abort() failed during __exit__ exception path: {e}");
             }
         }
@@ -308,13 +307,27 @@ impl PyXetUploadCommit {
     ///
     /// Releases the GIL while waiting, polling for ``KeyboardInterrupt`` every
     /// 100 ms so that Ctrl-C is delivered promptly.
+    ///
+    /// When a progress callback was registered, it is invoked once more on the
+    /// calling thread with the final snapshot after all work completes.
     pub fn wait_to_finish(&self, py: Python<'_>) -> PyResult<XetCommitReport> {
         let inner = self.inner.clone();
-        blocking_call_with_signal_check(py, move || inner.commit_blocking())
+        let result = blocking_call_with_signal_check(py, move || inner.commit_blocking());
+        if let (Some(handles), Some(progress)) = (&self.upload_handles, &self.background_progress) {
+            // ignore any error from progress update
+            let _ = progress.stop_and_emit(py, || {
+                let item_reports = item_reports_from_upload_handles(handles);
+                (self.inner.progress(), item_reports)
+            });
+        }
+        result
     }
 
     /// Cancel all active uploads in this commit.
     pub fn abort(&self) -> PyResult<()> {
+        if let Some(progress) = &self.background_progress {
+            progress.stop_and_join()?;
+        }
         self.inner.abort().map_err(convert_xet_error)
     }
 
@@ -408,6 +421,7 @@ mod tests {
                 .build_blocking()
                 .unwrap(),
             upload_handles: None,
+            background_progress: None,
         };
 
         Python::attach(|py| {
@@ -437,6 +451,7 @@ mod tests {
                 .build_blocking()
                 .unwrap(),
             upload_handles: None,
+            background_progress: None,
         };
 
         Python::attach(|py| {