pyturso 0.4.1__tar.gz → 0.4.2__tar.gz

{pyturso-0.4.1 → pyturso-0.4.2}/Cargo.lock

@@ -896,7 +896,7 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
 [[package]]
 name = "core_tester"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "anyhow",
  "assert_cmd",
@@ -2729,7 +2729,7 @@ dependencies = [
 
 [[package]]
 name = "limbo_completion"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "mimalloc",
  "turso_ext",
@@ -2737,7 +2737,7 @@ dependencies = [
 
 [[package]]
 name = "limbo_crypto"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "blake3",
  "data-encoding",
@@ -2750,7 +2750,7 @@ dependencies = [
 
 [[package]]
 name = "limbo_csv"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "csv",
  "mimalloc",
@@ -2760,7 +2760,7 @@ dependencies = [
 
 [[package]]
 name = "limbo_fuzzy"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "mimalloc",
  "turso_ext",
@@ -2768,7 +2768,7 @@ dependencies = [
 
 [[package]]
 name = "limbo_ipaddr"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "ipnetwork",
  "mimalloc",
@@ -2777,7 +2777,7 @@ dependencies = [
 
 [[package]]
 name = "limbo_percentile"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "mimalloc",
  "turso_ext",
@@ -2785,7 +2785,7 @@ dependencies = [
 
 [[package]]
 name = "limbo_regexp"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "mimalloc",
  "regex",
@@ -2794,7 +2794,7 @@ dependencies = [
 
 [[package]]
 name = "limbo_sim"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "anyhow",
  "bitflags 2.9.4",
@@ -2831,7 +2831,7 @@ dependencies = [
 
 [[package]]
 name = "limbo_sqlite_test_ext"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "cc",
 ]
@@ -3718,7 +3718,7 @@ dependencies = [
 
 [[package]]
 name = "py-turso"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "anyhow",
  "pyo3",
@@ -4595,7 +4595,7 @@ checksum = "d372029cb5195f9ab4e4b9aef550787dce78b124fcaee8d82519925defcd6f0d"
 
 [[package]]
 name = "sql_generation"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "anarchist-readable-name-generator-lib",
  "anyhow",
@@ -5299,7 +5299,7 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
 
 [[package]]
 name = "turso"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "anyhow",
  "bytes",
@@ -5323,14 +5323,14 @@ dependencies = [
 
 [[package]]
 name = "turso-dotnet"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "turso_core",
 ]
 
 [[package]]
 name = "turso-java"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "jni",
  "thiserror 2.0.16",
@@ -5339,7 +5339,7 @@ dependencies = [
 
 [[package]]
 name = "turso_cli"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "anyhow",
  "bytes",
@@ -5380,7 +5380,7 @@ dependencies = [
 
 [[package]]
 name = "turso_core"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "aegis",
  "aes",
@@ -5447,7 +5447,7 @@ dependencies = [
 
 [[package]]
 name = "turso_dart"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "flutter_rust_bridge",
  "turso_core",
@@ -5455,7 +5455,7 @@ dependencies = [
 
 [[package]]
 name = "turso_ext"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "chrono",
  "getrandom 0.3.2",
@@ -5464,7 +5464,7 @@ dependencies = [
 
 [[package]]
 name = "turso_ext_tests"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "env_logger 0.11.7",
  "lazy_static",
@@ -5475,7 +5475,7 @@ dependencies = [
 
 [[package]]
 name = "turso_macros"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5484,7 +5484,7 @@ dependencies = [
 
 [[package]]
 name = "turso_node"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "chrono",
  "napi",
@@ -5497,7 +5497,7 @@ dependencies = [
 
 [[package]]
 name = "turso_parser"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "bitflags 2.9.4",
  "criterion",
@@ -5514,7 +5514,7 @@ dependencies = [
 
 [[package]]
 name = "turso_sdk_kit"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "bindgen",
  "env_logger 0.11.7",
@@ -5527,7 +5527,7 @@ dependencies = [
 
 [[package]]
 name = "turso_sdk_kit_macros"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5536,7 +5536,7 @@ dependencies = [
 
 [[package]]
 name = "turso_sqlite3"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "env_logger 0.11.7",
  "libc",
@@ -5549,7 +5549,7 @@ dependencies = [
 
 [[package]]
 name = "turso_stress"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "antithesis_sdk",
  "clap",
@@ -5565,7 +5565,7 @@ dependencies = [
 
 [[package]]
 name = "turso_sync_engine"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "base64 0.22.1",
  "bytes",
@@ -5592,7 +5592,7 @@ dependencies = [
 
 [[package]]
 name = "turso_sync_js"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "genawaiter",
  "napi",
@@ -5607,7 +5607,7 @@ dependencies = [
 
 [[package]]
 name = "turso_sync_sdk_kit"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "bindgen",
  "env_logger 0.11.7",
@@ -5624,7 +5624,7 @@ dependencies = [
 
 [[package]]
 name = "turso_whopper"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "anyhow",
  "clap",

{pyturso-0.4.1 → pyturso-0.4.2}/Cargo.toml

@@ -6,24 +6,24 @@ members = ["bindings/python"]
 exclude = ["perf/latency/limbo", "turso-test-runner"]
 
 [workspace.package]
-version = "0.4.1"
+version = "0.4.2"
 authors = ["the Limbo authors"]
 edition = "2021"
 license = "MIT"
 repository = "https://github.com/tursodatabase/turso"
 
 [workspace.dependencies]
-turso = { path = "bindings/rust", version = "0.4.1" }
-turso_node = { path = "bindings/javascript", version = "0.4.1" }
-turso_sdk_kit = { path = "sdk-kit", version = "0.4.1" }
-turso_sdk_kit_macros = { path = "sdk-kit-macros", version = "0.4.1" }
-turso_sync_sdk_kit = { path = "sync/sdk-kit", version = "0.4.1" }
-limbo_completion = { path = "extensions/completion", version = "0.4.1" }
-turso_core = { path = "core", version = "0.4.1" }
-turso_sync_engine = { path = "sync/engine", version = "0.4.1" }
-turso_ext = { path = "extensions/core", version = "0.4.1" }
-turso_macros = { path = "macros", version = "0.4.1" }
-turso_parser = { path = "parser", version = "0.4.1" }
+turso = { path = "bindings/rust", version = "0.4.2" }
+turso_node = { path = "bindings/javascript", version = "0.4.2" }
+turso_sdk_kit = { path = "sdk-kit", version = "0.4.2" }
+turso_sdk_kit_macros = { path = "sdk-kit-macros", version = "0.4.2" }
+turso_sync_sdk_kit = { path = "sync/sdk-kit", version = "0.4.2" }
+limbo_completion = { path = "extensions/completion", version = "0.4.2" }
+turso_core = { path = "core", version = "0.4.2" }
+turso_sync_engine = { path = "sync/engine", version = "0.4.2" }
+turso_ext = { path = "extensions/core", version = "0.4.2" }
+turso_macros = { path = "macros", version = "0.4.2" }
+turso_parser = { path = "parser", version = "0.4.2" }
 sql_generation = { path = "sql_generation" }
 strum = { version = "0.26", features = ["derive"] }
 strum_macros = "0.26"

{pyturso-0.4.1 → pyturso-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyturso
-Version: 0.4.1
+Version: 0.4.2
 Classifier: Development Status :: 3 - Alpha
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3

{pyturso-0.4.1 → pyturso-0.4.2}/core/mvcc/cursor.rs

@@ -437,8 +437,7 @@ impl<Clock: LogicalClock + 'static> MvccLazyCursor<Clock> {
     }
 
     fn is_btree_allocated(&self) -> bool {
-        let maybe_root_page = self.db.table_id_to_rootpage.get(&self.table_id);
-        maybe_root_page.is_some_and(|entry| entry.value().is_some())
+        self.db.is_btree_allocated(&self.table_id)
     }
 
     fn query_btree_version_is_valid(&self, key: &RowKey) -> bool {

{pyturso-0.4.1 → pyturso-0.4.2}/core/mvcc/database/mod.rs

@@ -2310,23 +2310,7 @@ impl<Clock: LogicalClock> MvStore<Clock> {
 
         for rowid in &tx.write_set {
             let rowid = rowid.value();
-            if let Some(row_versions) = self.rows.get(rowid) {
-                let mut row_versions = row_versions.value().write();
-                for rv in row_versions.iter_mut() {
-                    if let Some(TxTimestampOrID::TxID(id)) = rv.begin {
-                        assert_eq!(id, tx_id);
-                        // If the transaction has aborted,
-                        // it marks all its new versions as garbage and sets their Begin
-                        // and End timestamps to infinity to make them invisible
-                        // See section 2.4: https://www.cs.cmu.edu/~15721-f24/papers/Hekaton.pdf
-                        rv.begin = None;
-                        rv.end = None;
-                    } else if rv.end == Some(TxTimestampOrID::TxID(tx_id)) {
-                        // undo deletions by this transaction
-                        rv.end = None;
-                    }
-                }
-            }
+            self.rollback_rowid(tx_id, rowid);
         }
 
         if connection.schema.read().schema_version > connection.db.schema.lock().schema_version {
@@ -2342,6 +2326,38 @@ impl<Clock: LogicalClock> MvStore<Clock> {
         self.remove_tx(tx_id);
     }
 
+    fn rollback_rowid(&self, tx_id: u64, rowid: &RowID) {
+        if rowid.row_id.is_int_key() {
+            self.rollback_table_rowid(tx_id, rowid);
+        } else {
+            self.rollback_index_rowid(tx_id, rowid);
+        }
+    }
+
+    fn rollback_index_rowid(&self, tx_id: u64, rowid: &RowID) {
+        if let Some(index) = self.index_rows.get(&rowid.table_id) {
+            let index = index.value();
+            let RowKey::Record(ref index_key) = rowid.row_id else {
+                panic!("Index writes must have a record key");
+            };
+            if let Some(row_versions) = index.get(index_key) {
+                let mut row_versions = row_versions.value().write();
+                for rv in row_versions.iter_mut() {
+                    rollback_row_version(tx_id, rv);
+                }
+            }
+        }
+    }
+
+    fn rollback_table_rowid(&self, tx_id: u64, rowid: &RowID) {
+        if let Some(row_versions) = self.rows.get(rowid) {
+            let mut row_versions = row_versions.value().write();
+            for rv in row_versions.iter_mut() {
+                rollback_row_version(tx_id, rv);
+            }
+        }
+    }
+
     /// Returns true if the given transaction is the exclusive transaction.
     #[inline]
     pub fn is_exclusive_tx(&self, tx_id: &TxID) -> bool {
@@ -2808,6 +2824,25 @@ impl<Clock: LogicalClock> MvStore<Clock> {
             allocator
         }
     }
+
+    pub fn is_btree_allocated(&self, table_id: &MVTableId) -> bool {
+        let maybe_root_page = self.table_id_to_rootpage.get(table_id);
+        maybe_root_page.is_some_and(|entry| entry.value().is_some())
+    }
+}
+
+fn rollback_row_version(tx_id: u64, rv: &mut RowVersion) {
+    if rv.begin == Some(TxTimestampOrID::TxID(tx_id)) {
+        // If the transaction has aborted,
+        // it marks all its new versions as garbage and sets their Begin
+        // and End timestamps to infinity to make them invisible
+        // See section 2.4: https://www.cs.cmu.edu/~15721-f24/papers/Hekaton.pdf
+        rv.begin = None;
+        rv.end = None;
+    } else if rv.end == Some(TxTimestampOrID::TxID(tx_id)) {
+        // undo deletions by this transaction
+        rv.end = None;
+    }
 }
 
 impl RowidAllocator {
@@ -2925,7 +2960,7 @@ fn is_begin_visible(txs: &SkipMap<TxID, Transaction>, tx: &Transaction, rv: &Row
         Some(TxTimestampOrID::TxID(rv_begin)) => {
             let tb = txs
                 .get(&rv_begin)
-                .expect("transaction should exist in txs map");
+                .unwrap_or_else(|| panic!("transaction {rv_begin:?} should exist in txs map"));
             let tb = tb.value();
             let visible = match tb.state.load() {
                 TransactionState::Active => tx.tx_id == tb.tx_id && rv.end.is_none(),

{pyturso-0.4.1 → pyturso-0.4.2}/core/mvcc/database/tests.rs

@@ -1956,3 +1956,53 @@ fn test_skips_updated_rowid() {
     assert_eq!(rows.len(), 1);
     assert_eq!(rows[0][1].as_int().unwrap(), 3);
 }
+
+#[test]
+fn test_mvcc_integrity_check() {
+    tracing_subscriber::fmt()
+        .with_ansi(false)
+        .with_max_level(tracing_subscriber::filter::LevelFilter::TRACE)
+        .init();
+    let db = MvccTestDbNoConn::new_with_random_db();
+    let conn = db.connect();
+
+    conn.execute("CREATE TABLE t(a INTEGER PRIMARY KEY)")
+        .unwrap();
+
+    // we insert with default values
+    conn.execute("INSERT INTO t values(1)").unwrap();
+
+    let ensure_integrity = || {
+        let rows = get_rows(&conn, "PRAGMA integrity_check");
+        assert_eq!(rows.len(), 1);
+        assert_eq!(&rows[0][0].cast_text().unwrap(), "ok");
+    };
+
+    ensure_integrity();
+
+    conn.execute("PRAGMA wal_checkpoint(TRUNCATE)").unwrap();
+
+    ensure_integrity();
+}
+
+#[test]
+fn test_rollback_with_index() {
+    let db = MvccTestDbNoConn::new_with_random_db();
+    let conn = db.connect();
+
+    conn.execute("CREATE TABLE t(a INTEGER PRIMARY KEY, b INTEGER UNIQUE)")
+        .unwrap();
+
+    // we insert with default values
+    conn.execute("BEGIN CONCURRENT").unwrap();
+    conn.execute("INSERT INTO t values (1, 1)").unwrap();
+    conn.execute("ROLLBACK").unwrap();
+
+    // This query will try to use index to find the row, if we rollback correctly it shouldn't panic
+    let rows = get_rows(&conn, "SELECT * FROM t where b = 1");
+    assert_eq!(rows.len(), 0);
+
+    let rows = get_rows(&conn, "PRAGMA integrity_check");
+    assert_eq!(rows.len(), 1);
+    assert_eq!(&rows[0][0].to_string(), "ok");
+}

{pyturso-0.4.1 → pyturso-0.4.2}/core/storage/btree.rs

@@ -25,7 +25,7 @@ use crate::{
         SeekResult,
     },
     util::IOExt,
-    Completion,
+    Completion, MvStore,
 };
 
 use crate::{
@@ -5800,8 +5800,28 @@ pub fn integrity_check(
     state: &mut IntegrityCheckState,
     errors: &mut Vec<IntegrityCheckError>,
     pager: &Arc<Pager>,
+    mv_store: Option<&Arc<MvStore>>,
 ) -> Result<IOResult<()>> {
+    if let Some(mv_store) = mv_store {
+        let Some(IntegrityCheckPageEntry {
+            page_idx: root_page,
+            ..
+        }) = state.page_stack.last().cloned()
+        else {
+            panic!("Page stack is empty on integrity_check start");
+        };
+        if root_page < 0 {
+            let table_id = mv_store.get_table_id_from_root_page(root_page);
+            turso_assert!(
+                !mv_store.is_btree_allocated(&table_id),
+                "we got a negative page index that is reported as allocated"
+            );
+            state.page_stack.pop();
+            return Ok(IOResult::Done(()));
+        }
+    }
     if state.db_size == 0 {
+        state.page_stack.pop();
         return Ok(IOResult::Done(()));
     }
     loop {
@@ -5814,6 +5834,10 @@ pub fn integrity_check(
         else {
             return Ok(IOResult::Done(()));
         };
+        turso_assert!(
+            page_idx >= 0,
+            "pages should be positive during integrity check"
+        );
         let page = match state.page.take() {
             Some(page) => page,
             None => {

{pyturso-0.4.1 → pyturso-0.4.2}/core/storage/pager.rs

@@ -959,6 +959,9 @@ enum CheckpointPhase {
     TruncateDbFile {
         sync_mode: crate::SyncMode,
         clear_page_cache: bool,
+        /// Whether we've invalidated page 1 from cache (needed because checkpoint may write
+        /// pages directly from WALto DB file, so cached page 1 of the checkpointer connection may have stale database_size)
+        page1_invalidated: bool,
     },
     /// Sync the database file after checkpoint (if sync_mode != Off and we backfilled any frames from the WAL).
     SyncDbFile { clear_page_cache: bool },
@@ -1092,15 +1095,27 @@ pub struct Savepoint {
     /// If the database grows during the savepoint and a rollback to the savepoint is performed,
     /// the pages exceeding the database size at the start of the savepoint will be ignored.
     db_size: AtomicU32,
+    /// We might want to rollback.
+    /// WAL max frame at the start of the savepoint.
+    wal_max_frame: AtomicU64,
+    /// WAL checksum at the start of the savepoint.
+    wal_checksum: RwLock<(u32, u32)>,
 }
 
 impl Savepoint {
-    pub fn new(subjournal_offset: u64, db_size: u32) -> Self {
+    pub fn new(
+        subjournal_offset: u64,
+        db_size: u32,
+        wal_max_frame: u64,
+        wal_checksum: (u32, u32),
+    ) -> Self {
         Self {
             start_offset: AtomicU64::new(subjournal_offset),
             write_offset: AtomicU64::new(subjournal_offset),
             page_bitmap: RwLock::new(RoaringBitmap::new()),
             db_size: AtomicU32::new(db_size),
+            wal_max_frame: AtomicU64::new(wal_max_frame),
+            wal_checksum: RwLock::new(wal_checksum),
         }
     }
 
@@ -1486,7 +1501,12 @@ impl Pager {
         // the subjournal offset should always be 0 as we should only have max 1 savepoint
         // opened at any given time.
         turso_assert!(subjournal_offset == 0, "subjournal offset should be 0");
-        let savepoint = Savepoint::new(subjournal_offset, db_size);
+        let (wal_max_frame, wal_checksum) = if let Some(wal) = &self.wal {
+            (wal.get_max_frame(), wal.get_last_checksum())
+        } else {
+            (0, (0, 0))
+        };
+        let savepoint = Savepoint::new(subjournal_offset, db_size, wal_max_frame, wal_checksum);
         let mut savepoints = self.savepoints.write();
         turso_assert!(
             savepoints.is_empty(),
@@ -1613,6 +1633,12 @@ impl Pager {
 
         self.page_cache.write().truncate(db_size as usize)?;
 
+        if let Some(wal) = &self.wal {
+            let wal_max_frame = savepoint.wal_max_frame.load(Ordering::SeqCst);
+            let wal_checksum = *savepoint.wal_checksum.read();
+            wal.rollback(Some(wal_max_frame), Some(wal_checksum));
+        }
+
         Ok(true)
     }
 
@@ -3336,10 +3362,16 @@ impl Pager {
     }
 
     fn reset_checkpoint_state(&self) {
+        self.clear_checkpoint_state();
+        self.commit_info.write().state = CommitState::PrepareWal;
+    }
+
+    /// Reset checkpoint state machine to initial state.
+    /// Use this to clean up after a failed explicit checkpoint (PRAGMA wal_checkpoint).
+    pub fn clear_checkpoint_state(&self) {
         let mut state = self.checkpoint_state.write();
         state.phase = CheckpointPhase::NotCheckpointing;
         state.result = None;
-        self.commit_info.write().state = CommitState::PrepareWal;
     }
 
     /// Clean up after a checkpoint failure. The WAL commit succeeded but checkpoint failed.
@@ -3398,6 +3430,7 @@ impl Pager {
                         state.phase = CheckpointPhase::TruncateDbFile {
                             sync_mode,
                             clear_page_cache,
+                            page1_invalidated: false,
                         };
                     } else if res.num_backfilled == 0 || sync_mode == crate::SyncMode::Off {
                         state.phase = CheckpointPhase::Finalize { clear_page_cache };
@@ -3409,6 +3442,7 @@ impl Pager {
                 CheckpointPhase::TruncateDbFile {
                     sync_mode,
                     clear_page_cache,
+                    page1_invalidated,
                 } => {
                     let should_skip_truncate = {
                         let state = self.checkpoint_state.read();
@@ -3426,6 +3460,18 @@ impl Pager {
                         }
                         continue;
                     }
+                    // Invalidate page 1 (header) in cache before reading - checkpoint potentially wrote pages
+                    // directly to DB file from the WAL, so the checkpointer connections' page 1 may have stale database_size.
+                    if !page1_invalidated {
+                        let page1_key = PageCacheKey::new(DatabaseHeader::PAGE_ID);
+                        self.page_cache.write().delete(page1_key)?;
+                        let mut state = self.checkpoint_state.write();
+                        state.phase = CheckpointPhase::TruncateDbFile {
+                            sync_mode,
+                            clear_page_cache,
+                            page1_invalidated: true,
+                        };
+                    }
 
                     // Truncate the database file unless already at correct size
                     let db_size =
@@ -3489,18 +3535,19 @@ impl Pager {
                 CheckpointPhase::Finalize { clear_page_cache } => {
                     let mut state = self.checkpoint_state.write();
                     let mut res = state.result.take().expect("result should be set");
-
-                    // Release checkpoint guard
-                    res.release_guard();
+                    state.phase = CheckpointPhase::NotCheckpointing;
 
                     // Clear page cache only if requested (explicit checkpoints do this, auto-checkpoint does not)
                     if clear_page_cache {
                         self.page_cache.write().clear(false).map_err(|e| {
+                            res.release_guard();
                             LimboError::InternalError(format!("Failed to clear page cache: {e:?}"))
                         })?;
                     }
 
-                    state.phase = CheckpointPhase::NotCheckpointing;
+                    // Release checkpoint guard
+                    res.release_guard();
+
                     return Ok(IOResult::Done(res));
                 }
             }
@@ -4109,7 +4156,7 @@ impl Pager {
         }
         if is_write {
             if let Some(wal) = self.wal.as_ref() {
-                wal.rollback();
+                wal.rollback(None, None);
             }
         }
     }