ducpy 3.0.4__tar.gz → 3.1.0__tar.gz

{ducpy-3.0.4 → ducpy-3.1.0}/Cargo.lock

@@ -288,9 +288,9 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
 
 [[package]]
 name = "crypto-common"
-version = "0.1.6"
+version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
+checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
 dependencies = [
  "generic-array",
  "typenum",
@@ -318,9 +318,9 @@ dependencies = [
 
 [[package]]
 name = "deranged"
-version = "0.5.4"
+version = "0.5.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a41953f86f8a05768a6cda24def994fd2f424b04ec5c719cf89989779f199071"
+checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
 dependencies = [
  "powerfmt",
 ]
@@ -364,6 +364,7 @@ dependencies = [
  "base64 0.21.7",
  "env_logger",
  "flate2",
+ "fossil-delta",
  "lazy_static",
  "log",
  "percent-encoding",
@@ -410,7 +411,7 @@ dependencies = [
 
 [[package]]
 name = "ducpy-native"
-version = "3.0.4"
+version = "3.1.0"
 dependencies = [
  "duc",
  "pyo3",
@@ -570,11 +571,17 @@ dependencies = [
  "ttf-parser 0.25.1",
 ]
 
+[[package]]
+name = "fossil-delta"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e42f72dc65ff480af54c73c63d4b425936aac4167a01b7c9e1ed15aacace22c"
+
 [[package]]
 name = "generic-array"
-version = "0.14.9"
+version = "0.14.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
 dependencies = [
  "typenum",
  "version_check",
@@ -706,9 +713,9 @@ checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
 
 [[package]]
 name = "hipdf"
-version = "1.3.3"
+version = "1.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d8e0025c137fd7f666ec73ba8a1c8c7f4940edb83fdb77013e5e12703204c80"
+checksum = "e3b986659f7ff291144addff668450799e4c2cd69686601cd48dfe23d7afc316"
 dependencies = [
  "jpeg-decoder",
  "lopdf",
@@ -873,24 +880,24 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
 
 [[package]]
 name = "jiff"
-version = "0.2.15"
+version = "0.2.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49"
+checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359"
 dependencies = [
  "jiff-static",
  "jiff-tzdb-platform",
  "log",
  "portable-atomic",
  "portable-atomic-util",
- "serde",
- "windows-sys 0.59.0",
+ "serde_core",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
 name = "jiff-static"
-version = "0.2.15"
+version = "0.2.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4"
+checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -899,9 +906,9 @@ dependencies = [
 
 [[package]]
 name = "jiff-tzdb"
-version = "0.1.4"
+version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1283705eb0a21404d2bfd6eef2a7593d240bc42a0bdb39db0ad6fa2ec026524"
+checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076"
 
 [[package]]
 name = "jiff-tzdb-platform"
@@ -1159,9 +1166,9 @@ dependencies = [
 
 [[package]]
 name = "num-conv"
-version = "0.1.0"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
+checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050"
 
 [[package]]
 name = "num-traits"
@@ -1315,9 +1322,9 @@ checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
 
 [[package]]
 name = "portable-atomic-util"
-version = "0.2.4"
+version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507"
+checksum = "7a9db96d7fa8782dd8c15ce32ffe8680bbd1e978a43bf51a34d39483540495f5"
 dependencies = [
  "portable-atomic",
 ]
@@ -1471,18 +1478,18 @@ dependencies = [
 
 [[package]]
 name = "rand_core"
-version = "0.9.3"
+version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
+checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
 dependencies = [
  "getrandom",
 ]
 
 [[package]]
 name = "rangemap"
-version = "1.6.0"
+version = "1.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f93e7e49bb0bf967717f7bd674458b3d6b0c5f48ec7e3038166026a69fc22223"
+checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68"
 
 [[package]]
 name = "rayon"
@@ -1971,30 +1978,30 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.44"
+version = "0.3.47"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d"
+checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c"
 dependencies = [
  "deranged",
  "itoa",
  "num-conv",
  "powerfmt",
- "serde",
+ "serde_core",
  "time-core",
  "time-macros",
 ]
 
 [[package]]
 name = "time-core"
-version = "0.1.6"
+version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b"
+checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca"
 
 [[package]]
 name = "time-macros"
-version = "0.2.24"
+version = "0.2.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3"
+checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215"
 dependencies = [
  "num-conv",
  "time-core",
@@ -2097,9 +2104,9 @@ checksum = "462eeb75aeb73aea900253ce739c8e18a67423fadf006037cd3ff27e82748a06"
 
 [[package]]
 name = "unicode-normalization"
-version = "0.1.24"
+version = "0.1.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956"
+checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
 dependencies = [
  "tinyvec",
 ]
@@ -2175,9 +2182,9 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
 
 [[package]]
 name = "wasip2"
-version = "1.0.1+wasi-0.2.4"
+version = "1.0.2+wasi-0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7"
+checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5"
 dependencies = [
  "wit-bindgen",
 ]
@@ -2422,9 +2429,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
 [[package]]
 name = "wit-bindgen"
-version = "0.46.0"
+version = "0.51.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59"
+checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"
 
 [[package]]
 name = "write-fonts"
@@ -2470,18 +2477,18 @@ dependencies = [
 
 [[package]]
 name = "zerocopy"
-version = "0.8.27"
+version = "0.8.41"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c"
+checksum = "96e13bc581734df6250836c59a5f44f3c57db9f9acb9dc8e3eaabdaf6170254d"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.27"
+version = "0.8.41"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831"
+checksum = "3545ea9e86d12ab9bba9fcd99b54c1556fd3199007def5a03c375623d05fac1c"
 dependencies = [
  "proc-macro2",
  "quote",

{ducpy-3.0.4 → ducpy-3.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ducpy
-Version: 3.0.4
+Version: 3.1.0
 Classifier: License :: OSI Approved :: MIT License
 Requires-Dist: nanoid>=2.0.0
 License-File: LICENSE

{ducpy-3.0.4 → ducpy-3.1.0}/packages/ducpy/crate/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ducpy-native"
-version = "3.0.4"
+version = "3.1.0"
 edition = "2021"
 description = "Python native extension for the duc file format (used by ducpy)"
 publish = false

{ducpy-3.0.4 → ducpy-3.1.0}/packages/ducrs/Cargo.toml

@@ -13,6 +13,7 @@ base64 = "0.21.5"
 percent-encoding = "2.3.1"
 log = "0.4.22"
 flate2 = "1.0"
+fossil-delta = "0.2"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_repr = "0.1"

{ducpy-3.0.4 → ducpy-3.1.0}/packages/ducrs/build.rs

@@ -43,7 +43,63 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         }
         println!("cargo:rerun-if-changed={}", src.display());
     }
+    // Scan schema/migrations/*.sql, parse (from, to) from filenames like
+    // "3000000_to_3000001.sql", and generate migrations_registry.rs with a
+    // sorted static array of (from_version, to_version, sql) tuples.
+    // Adding a new migration requires only a new SQL file — no Rust changes.
+    let migrations_dir = schema_dir.join("migrations");
+    let mut migrations: Vec<(i64, i64, String)> = Vec::new();
 
+    if migrations_dir.is_dir() {
+        for entry in fs::read_dir(&migrations_dir)? {
+            let entry = entry?;
+            let path = entry.path();
+            if path.extension().and_then(|e| e.to_str()) != Some("sql") {
+                continue;
+            }
+            let stem = match path.file_stem().and_then(|s| s.to_str()) {
+                Some(s) => s.to_owned(),
+                None => continue,
+            };
+            // Expect exactly "<from>_to_<to>".
+            if let Some((from_str, to_str)) = stem.split_once("_to_") {
+                if let (Ok(from), Ok(to)) = (from_str.parse::<i64>(), to_str.parse::<i64>()) {
+                    let sql = fs::read_to_string(&path)?;
+                    migrations.push((from, to, sql));
+                    println!("cargo:rerun-if-changed={}", path.display());
+                } else {
+                    eprintln!("cargo:warning=Skipping migration file with non-integer versions: {stem}.sql");
+                }
+            } else {
+                eprintln!("cargo:warning=Skipping migration file with unexpected name: {stem}.sql");
+            }
+        }
+        println!("cargo:rerun-if-changed={}", migrations_dir.display());
+    }
+
+    // Sort ascending by from_version so chained migrations apply in order.
+    migrations.sort_by_key(|(from, _, _)| *from);
+
+    // Emit migrations_registry.rs: a static array included by bootstrap.rs.
+    let mut reg = String::from(
+        "// Auto-generated by build.rs — do not edit by hand.\n\
+         // Each entry: (from_version, to_version, sql).\n\
+         pub(crate) static MIGRATIONS: &[(i64, i64, &str)] = &[\n",
+    );
+    for (from, to, sql) in &migrations {
+        reg.push_str(&format!("    ({from}i64, {to}i64, \""));
+        for c in sql.chars() {
+            match c {
+                '"' => reg.push_str("\\\""),
+                '\\' => reg.push_str("\\\\"),
+                '\r' => {} // normalise CRLF → LF
+                c => reg.push(c),
+            }
+        }
+        reg.push_str("\"),\n");
+    }
+    reg.push_str("];\n");
+    fs::write(out_dir.join("migrations_registry.rs"), reg)?;
     let sql_path = schema_dir.join("duc.sql");
     let (user_version, semver_version) = match fs::read_to_string(&sql_path) {
         Ok(sql) => {

{ducpy-3.0.4 → ducpy-3.1.0}/packages/ducrs/src/api/version_control.rs

@@ -14,10 +14,12 @@ use flate2::read::ZlibDecoder;
 use flate2::write::ZlibEncoder;
 use flate2::Compression;
 use rusqlite::OptionalExtension;
-use std::io::{Read, Write};
+use std::io::Read;
+use std::io::Write;
 use std::os::raw::c_char;
 
 use crate::db::{DbError, DbResult, DucConnection};
+use crate::parse::{decompress_duc_bytes, is_sqlite_header};
 use crate::types::{
     Checkpoint, Delta, SchemaMigration, VersionBase, VersionChain, VersionGraph,
     VersionGraphMetadata,
@@ -132,10 +134,11 @@ impl<'a> VersionControl<'a> {
 
     /// Restore the document state at *exactly* `version_number`.
     ///
-    /// Algorithm (matches the SQL schema comments):
+    /// Algorithm:
     /// 1. If a checkpoint exists at that version → return its `data` directly.
-    /// 2. Otherwise find the nearest checkpoint with `version_number <= target`
-    ///    in the same schema version, then replay deltas sequentially.
+    /// 2. Otherwise find the delta at that version, load its base checkpoint,
+    ///    and decode the changeset (handles both legacy full-snapshot and
+    ///    modern XOR diff formats transparently).
     pub fn restore_version(&self, version_number: i64) -> DbResult<RestoredVersion> {
         self.conn.with(|c| {
             // 1) Try direct checkpoint hit
@@ -172,7 +175,7 @@ impl<'a> VersionControl<'a> {
                 )
                 .map_err(DbError::from)?;
 
-            // 3) Load the base checkpoint
+            // 3) Load the base checkpoint data
             let base_data: Vec<u8> = c
                 .query_row(
                     "SELECT data FROM checkpoints WHERE id = ?1",
@@ -181,48 +184,17 @@ impl<'a> VersionControl<'a> {
                 )
                 .map_err(DbError::from)?;
 
-            let base_cp_version: i64 = c
+            // 4) Load the target delta's changeset
+            let target_changeset: Vec<u8> = c
                 .query_row(
-                    "SELECT version_number FROM checkpoints WHERE id = ?1",
-                    [&target_base_cp_id],
+                    "SELECT changeset FROM deltas WHERE version_number = ?1",
+                    [version_number],
                     |row| row.get(0),
                 )
                 .map_err(DbError::from)?;
 
-            // 4) Collect and replay deltas from base_checkpoint up to target version
-            let mut stmt = c
-                .prepare(
-                    "SELECT changeset FROM deltas
-                     WHERE base_checkpoint_id = ?1
-                       AND schema_version = ?2
-                       AND version_number > ?3
-                       AND version_number <= ?4
-                     ORDER BY delta_sequence ASC",
-                )
-                .map_err(DbError::from)?;
-
-            let delta_payloads: Vec<Vec<u8>> = stmt
-                .query_map(
-                    rusqlite::params![target_base_cp_id, target_sv, base_cp_version, version_number],
-                    |row| row.get::<_, Vec<u8>>(0),
-                )
-                .map_err(DbError::from)?
-                .collect::<Result<Vec<_>, _>>()
-                .map_err(DbError::from)?;
-
-            // 5) Apply deltas sequentially. Each delta payload is zlib-compressed.
-            //    The application-level delta format is opaque — the caller is
-            //    responsible for interpreting the bytes. What we do here is
-            //    decompress each payload and return the *final* state.
-            //
-            //    For snapshot-based deltas (our current model), each delta
-            //    payload IS the full state at that version (compressed).
-            //    So we just need the *last* payload.
-            let final_data = if let Some(last_changeset) = delta_payloads.last() {
-                decompress_zlib(last_changeset)?
-            } else {
-                base_data
-            };
+            // 5) Decode: auto-detects v3 (bsdiff) / v2 (XOR) / v1 (legacy snapshot)
+            let final_data = apply_delta_changeset(&base_data, &target_changeset)?;
 
             Ok(RestoredVersion {
                 version_number,
@@ -411,6 +383,10 @@ impl<'a> VersionControl<'a> {
     /// Insert a new delta into the database and update the version graph
     /// singleton row.
     ///
+    /// `delta.payload` must be the **full document state** at this version
+    /// (uncompressed). The method automatically computes a fossil delta
+    /// against the base checkpoint, producing a compact changeset.
+    ///
     /// If the delta's `schema_version` is higher than the stored
     /// `current_schema_version`, the migration bookkeeping is performed
     /// automatically.
@@ -429,8 +405,18 @@ impl<'a> VersionControl<'a> {
                 )
                 .map_err(DbError::from)?;
 
-            // Compress the payload with zlib before storing
-            let compressed = compress_zlib(&delta.payload)?;
+            // Load the base checkpoint data for delta computation
+            let base_data: Vec<u8> = c
+                .query_row(
+                    "SELECT data FROM checkpoints WHERE id = ?1",
+                    [&delta.base_checkpoint_id],
+                    |row| row.get(0),
+                )
+                .map_err(DbError::from)?;
+
+            // Compute checkpoint-relative fossil delta changeset
+            let changeset = create_bsdiff_changeset(&base_data, &delta.payload)?;
+            let stored_size = changeset.len() as i64;
 
             c.execute(
                 "INSERT OR REPLACE INTO deltas
@@ -450,8 +436,8 @@ impl<'a> VersionControl<'a> {
                     delta.base.description,
                     delta.base.is_manual_save as i32,
                     delta.base.user_id,
-                    compressed,
-                    delta.size_bytes,
+                    changeset,
+                    stored_size,
                 ],
             )
             .map_err(DbError::from)?;
@@ -983,15 +969,129 @@ fn decompress_zlib(compressed: &[u8]) -> DbResult<Vec<u8>> {
     Ok(decompressed)
 }
 
-/// Compress a blob with zlib.
-fn compress_zlib(data: &[u8]) -> DbResult<Vec<u8>> {
+/// Compress a blob with zlib (legacy v1 snapshot-compatible payload).
+fn compress_zlib(raw: &[u8]) -> DbResult<Vec<u8>> {
     let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
     encoder
-        .write_all(data)
+        .write_all(raw)
         .map_err(|e| DbError::Bootstrap(format!("zlib compression failed: {e}")))?;
     encoder
         .finish()
-        .map_err(|e| DbError::Bootstrap(format!("zlib compression finish failed: {e}")))
+        .map_err(|e| DbError::Bootstrap(format!("zlib finalize failed: {e}")))
+}
+
+/// Ensure the input is raw (uncompressed) SQLite bytes.
+///
+/// `.duc` files produced by `serializeDuc` are deflate-compressed. This
+/// helper transparently inflates them so that fossil delta operates on the
+/// raw SQLite pages — producing compact patches. If the input is already
+/// raw SQLite, it is returned as-is (zero-copy via `Cow`).
+fn ensure_raw_sqlite(buf: &[u8]) -> DbResult<std::borrow::Cow<'_, [u8]>> {
+    if is_sqlite_header(buf) {
+        Ok(std::borrow::Cow::Borrowed(buf))
+    } else {
+        let raw = decompress_duc_bytes(buf)
+            .map_err(|e| DbError::Bootstrap(format!("failed to decompress .duc blob: {e}")))?;
+        if !is_sqlite_header(&raw) {
+            return Err(DbError::Bootstrap(
+                "decompressed blob is not a valid SQLite database".into(),
+            ));
+        }
+        Ok(std::borrow::Cow::Owned(raw))
+    }
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+// Fossil delta encoding — checkpoint-relative structural diffing
+// ────────────────────────────────────────────────────────────────────────────
+//
+// Format (v5 — fossil delta):
+//   [0x44 'D'][0x46 'F']          – magic bytes ("DF" = Delta Fossil)
+//   [0x05]                        – format version
+//   [4 bytes LE u32]              – raw SQLite length of new state
+//   [remaining bytes]             – zlib-compressed fossil delta
+//
+// The fossil delta algorithm uses rolling checksums and emits granular
+// COPY (reference old bytes) + INSERT (literal new bytes) commands.
+// This is far more compact than page-level diffs because a 4KB page
+// with 20 changed bytes only stores those 20 bytes, not the full page.
+//
+// Fallback: if the delta is larger than a zlib-compressed full snapshot,
+// the snapshot is stored directly (no magic header → detected as raw zlib).
+
+/// Magic header identifying a fossil delta changeset.
+const DELTA_MAGIC_FOSSIL: [u8; 2] = [0x44, 0x46]; // "DF"
+const DELTA_FORMAT_V5: u8 = 5;
+
+/// Header: magic(2) + version(1) + new_len(4) = 7 bytes.
+const FOSSIL_HEADER_SIZE: usize = 2 + 1 + 4;
+
+/// Returns `true` if the blob starts with the fossil v5 magic header.
+fn is_fossil_format(changeset: &[u8]) -> bool {
+    changeset.len() >= FOSSIL_HEADER_SIZE
+        && changeset[0] == DELTA_MAGIC_FOSSIL[0]
+        && changeset[1] == DELTA_MAGIC_FOSSIL[1]
+        && changeset[2] == DELTA_FORMAT_V5
+}
+
+/// Compute a checkpoint-relative changeset using fossil delta.
+///
+/// Both inputs are transparently decompressed to raw SQLite bytes,
+/// then a fossil delta is computed and zlib-compressed.
+/// Falls back to a full zlib snapshot if the delta isn't smaller.
+pub fn create_bsdiff_changeset(base: &[u8], current: &[u8]) -> DbResult<Vec<u8>> {
+    let raw_base = ensure_raw_sqlite(base)?;
+    let raw_current = ensure_raw_sqlite(current)?;
+
+    // fossil_delta::delta(target, source) — target is what we want to reconstruct,
+    // source is what we already have. apply(source, delta) → target.
+    let raw_delta = fossil_delta::delta(&raw_current, &raw_base);
+    let compressed_delta = compress_zlib(&raw_delta)?;
+
+    let new_len = raw_current.len() as u32;
+    let mut encoded = Vec::with_capacity(FOSSIL_HEADER_SIZE + compressed_delta.len());
+    encoded.extend_from_slice(&DELTA_MAGIC_FOSSIL);
+    encoded.push(DELTA_FORMAT_V5);
+    encoded.extend_from_slice(&new_len.to_le_bytes());
+    encoded.extend_from_slice(&compressed_delta);
+
+    // Fallback: full zlib snapshot if delta isn't beneficial
+    let snapshot = compress_zlib(&raw_current)?;
+
+    if encoded.len() < snapshot.len() {
+        Ok(encoded)
+    } else {
+        Ok(snapshot)
+    }
+}
+
+/// Apply a fossil delta changeset to reconstruct the document state.
+///
+/// `base` is transparently decompressed if compressed.
+/// Returns raw (uncompressed) SQLite bytes.
+fn apply_fossil_changeset(base: &[u8], changeset: &[u8]) -> DbResult<Vec<u8>> {
+    let raw_base = ensure_raw_sqlite(base)?;
+
+    let compressed_delta = &changeset[FOSSIL_HEADER_SIZE..];
+    let raw_delta = decompress_zlib(compressed_delta)?;
+
+    fossil_delta::apply(&raw_base, &raw_delta)
+        .map_err(|e| DbError::Bootstrap(format!("fossil delta apply failed: {e:?}")))
+}
+
+/// Decode a stored changeset.
+///
+/// Detects fossil delta (v5) by magic header. Anything else is treated as
+/// a zlib-compressed full snapshot (fallback).
+///
+/// Returns raw (uncompressed) SQLite bytes.
+pub fn apply_delta_changeset(base_data: &[u8], changeset: &[u8]) -> DbResult<Vec<u8>> {
+    if is_fossil_format(changeset) {
+        apply_fossil_changeset(base_data, changeset)
+    } else {
+        // Snapshot fallback: zlib-compressed full state
+        decompress_zlib(changeset)
+    }
 }
 
 /// Generate a nanoid-style ID (22-char URL-safe random string).

{ducpy-3.0.4 → ducpy-3.1.0}/packages/ducrs/src/db/bootstrap.rs

@@ -18,6 +18,24 @@ const SEARCH_SCHEMA: &str = include_str!(concat!(env!("OUT_DIR"), "/search.sql")
 /// Expected `application_id` written by `duc.sql`.
 const APP_ID: i64 = 1_146_569_567; // "DUC_" in ASCII
 
+/// Schema version read from `duc.sql` at compile time by `build.rs`.
+/// Never hardcode this — it is always in sync with the SQL file.
+const CURRENT_VERSION: i64 = {
+    // build.rs emits DUC_SCHEMA_USER_VERSION as a decimal integer string.
+    let bytes = env!("DUC_SCHEMA_USER_VERSION").as_bytes();
+    let mut val: i64 = 0;
+    let mut i = 0;
+    while i < bytes.len() {
+        val = val * 10 + (bytes[i] - b'0') as i64;
+        i += 1;
+    }
+    val
+};
+
+// Migration registry auto-generated by build.rs from `schema/migrations/*.sql`.
+// Each entry is (from_version, to_version, sql). Sorted ascending by from_version.
+include!(concat!(env!("OUT_DIR"), "/migrations_registry.rs"));
+
 /// Per-connection PRAGMAs that must be re-applied on every open (they are
 /// either ephemeral or take effect only after the first statement on the
 /// connection).
@@ -57,8 +75,35 @@ pub(crate) fn bootstrap(conn: &Connection) -> Result<(), DbError> {
                 "unexpected application_id after bootstrap: {app_id} (expected {APP_ID})"
             )));
         }
+    } else if user_version == CURRENT_VERSION {
+        // Current schema — just ensure per-connection pragmas are active.
+        conn.execute_batch(CONN_PRAGMAS)
+            .map_err(|e| DbError::Bootstrap(format!("pragma apply failed: {e}")))?;
     } else {
-        // Existing database — just ensure per-connection pragmas are active.
+        // Walk the migration chain until we reach CURRENT_VERSION.
+        // build.rs generates MIGRATIONS sorted by from_version, so chaining
+        // (e.g. 3000000→3000001→3000002) works without any code changes here.
+        let mut current = user_version;
+        loop {
+            match MIGRATIONS.iter().find(|(from, _, _)| *from == current) {
+                Some((from, to, sql)) => {
+                    conn.execute_batch(sql).map_err(|e| {
+                        DbError::Bootstrap(format!("migration {from}\u{2192}{to} failed: {e}"))
+                    })?;
+                    // Re-read the version the migration SQL set via PRAGMA user_version.
+                    current =
+                        conn.pragma_query_value(None, "user_version", |r| r.get(0))?;
+                    if current == CURRENT_VERSION {
+                        break;
+                    }
+                }
+                None => {
+                    return Err(DbError::Bootstrap(format!(
+                        "unsupported schema version {user_version}; expected 0 or {CURRENT_VERSION}"
+                    )));
+                }
+            }
+        }
         conn.execute_batch(CONN_PRAGMAS)
             .map_err(|e| DbError::Bootstrap(format!("pragma apply failed: {e}")))?;
     }

{ducpy-3.0.4 → ducpy-3.1.0}/packages/ducrs/src/parse.rs

@@ -121,48 +121,131 @@ pub fn parse_lazy(buf: &[u8]) -> ParseResult<ExportedDataState> {
 }
 
 /// Fetch a single external file from a `.duc` buffer by file ID.
-pub fn get_external_file(buf: &[u8], file_id: &str) -> ParseResult<Option<DucExternalFileEntry>> {
+pub fn get_external_file(buf: &[u8], file_id: &str) -> ParseResult<Option<DucExternalFile>> {
     let conn = load_db_bytes(buf)?;
-    let mut stmt = conn.prepare(
-        "SELECT id, mime_type, data, created, last_retrieved, version FROM external_files WHERE id = ?1"
-    )?;
-    let result = stmt.query_row(params![file_id], |row| {
-        let id: String = row.get(0)?;
-        Ok(DucExternalFileEntry {
-            key: id.clone(),
-            value: DucExternalFileData {
+    let has_revisions_table: bool = conn.prepare(
+        "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='external_file_revisions'"
+    )?.query_row([], |row| row.get::<_, i32>(0)).unwrap_or(0) > 0;
+
+    if has_revisions_table {
+        // New schema: load file header + revisions.
+        let file = conn.prepare(
+            "SELECT id, active_revision_id, updated, version FROM external_files WHERE id = ?1"
+        )?.query_row(params![file_id], |row| {
+            Ok(DucExternalFile {
+                id:                 row.get(0)?,
+                active_revision_id: row.get(1)?,
+                updated:            row.get(2)?,
+                revisions:          HashMap::new(),
+                version:            row.get(3)?,
+            })
+        });
+        let mut file = match file {
+            Ok(f) => f,
+            Err(rusqlite::Error::QueryReturnedNoRows) => return Ok(None),
+            Err(e) => return Err(e.into()),
+        };
+
+        let mut rev_stmt = conn.prepare(
+            "SELECT id, size_bytes, checksum, source_name, mime_type, message,
+                    created, last_retrieved, data
+             FROM external_file_revisions WHERE file_id = ?1"
+        )?;
+        let rev_rows = rev_stmt.query_map(params![file_id], |row| {
+            let id: String = row.get(0)?;
+            Ok((id.clone(), ExternalFileRevision {
                 id,
-                mime_type: row.get(1)?,
-                data: row.get(2)?,
-                created: row.get(3)?,
-                last_retrieved: row.get(4)?,
+                size_bytes:     row.get(1)?,
+                checksum:       row.get(2)?,
+                source_name:    row.get(3)?,
+                mime_type:      row.get(4)?,
+                message:        row.get(5)?,
+                created:        row.get(6)?,
+                last_retrieved: row.get(7)?,
+                data:           row.get(8)?,
+            }))
+        })?;
+        for row in rev_rows {
+            let (rev_id, rev) = row?;
+            file.revisions.insert(rev_id, rev);
+        }
+        Ok(Some(file))
+    } else {
+        // Legacy schema: wrap flat row into a single-revision DucExternalFile.
+        let result = conn.prepare(
+            "SELECT id, mime_type, data, created, last_retrieved, version FROM external_files WHERE id = ?1"
+        )?.query_row(params![file_id], |row| {
+            let id: String = row.get(0)?;
+            let rev_id = format!("{}_rev1", id);
+            Ok(DucExternalFile {
+                id: id.clone(),
+                active_revision_id: rev_id.clone(),
+                updated: row.get(3)?,
                 version: row.get(5)?,
-            },
-        })
-    });
-    match result {
-        Ok(entry) => Ok(Some(entry)),
-        Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
-        Err(e) => Err(e.into()),
+                revisions: {
+                    let mut revs = HashMap::new();
+                    revs.insert(rev_id.clone(), ExternalFileRevision {
+                        id: rev_id,
+                        size_bytes: 0,
+                        checksum: None,
+                        source_name: None,
+                        mime_type: row.get(1)?,
+                        message: None,
+                        created: row.get(3)?,
+                        last_retrieved: row.get(4)?,
+                        data: row.get(2)?,
+                    });
+                    revs
+                },
+            })
+        });
+        match result {
+            Ok(file) => Ok(Some(file)),
+            Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
+            Err(e) => Err(e.into()),
+        }
     }
 }
 
-/// List metadata for all external files (without the heavy data blobs).
-pub fn list_external_files(buf: &[u8]) -> ParseResult<Vec<ExternalFileMetadata>> {
+/// List metadata summary for all external files (no data blobs loaded).
+pub fn list_external_files(buf: &[u8]) -> ParseResult<Vec<ExternalFileMeta>> {
     let conn = load_db_bytes(buf)?;
-    let mut stmt = conn.prepare(
-        "SELECT id, mime_type, created, last_retrieved, version FROM external_files"
-    )?;
-    let files: Vec<ExternalFileMetadata> = stmt.query_map([], |row| {
-        Ok(ExternalFileMetadata {
-            id: row.get(0)?,
-            mime_type: row.get(1)?,
-            created: row.get(2)?,
-            last_retrieved: row.get(3)?,
-            version: row.get(4)?,
-        })
-    })?.collect::<Result<Vec<_>, _>>()?;
-    Ok(files)
+    let has_revisions_table: bool = conn.prepare(
+        "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='external_file_revisions'"
+    )?.query_row([], |row| row.get::<_, i32>(0)).unwrap_or(0) > 0;
+
+    if has_revisions_table {
+        let mut stmt = conn.prepare(
+            "SELECT f.id, r.mime_type, r.created, r.last_retrieved, f.version
+             FROM external_files f
+             JOIN external_file_revisions r ON r.id = f.active_revision_id"
+        )?;
+        let files: Vec<ExternalFileMeta> = stmt.query_map([], |row| {
+            Ok(ExternalFileMeta {
+                id:             row.get(0)?,
+                mime_type:      row.get(1)?,
+                created:        row.get(2)?,
+                last_retrieved: row.get(3)?,
+                version:        row.get(4)?,
+            })
+        })?.collect::<Result<Vec<_>, _>>()?;
+        Ok(files)
+    } else {
+        // Legacy schema flat table.
+        let mut stmt = conn.prepare(
+            "SELECT id, mime_type, created, last_retrieved, version FROM external_files"
+        )?;
+        let files: Vec<ExternalFileMeta> = stmt.query_map([], |row| {
+            Ok(ExternalFileMeta {
+                id:             row.get(0)?,
+                mime_type:      row.get(1)?,
+                created:        row.get(2)?,
+                last_retrieved: row.get(3)?,
+                version:        row.get(4)?,
+            })
+        })?.collect::<Result<Vec<_>, _>>()?;
+        Ok(files)
+    }
 }
 
 // ─── Database import ─────────────────────────────────────────────────────────
@@ -223,12 +306,12 @@ fn load_db_bytes(buf: &[u8]) -> ParseResult<Connection> {
 }
 
 #[inline]
-fn is_sqlite_header(buf: &[u8]) -> bool {
+pub fn is_sqlite_header(buf: &[u8]) -> bool {
     buf.len() >= SQLITE_HEADER_MAGIC.len() && &buf[..SQLITE_HEADER_MAGIC.len()] == SQLITE_HEADER_MAGIC
 }
 
 /// Inflate a compressed deflate payload.
-fn decompress_duc_bytes(compressed: &[u8]) -> ParseResult<Vec<u8>> {
+pub fn decompress_duc_bytes(compressed: &[u8]) -> ParseResult<Vec<u8>> {
     use flate2::read::DeflateDecoder;
     use std::io::Read;
 
@@ -1685,27 +1768,131 @@ fn read_hatch_pattern_lines(conn: &Connection, owner_type: &str, owner_id: i64)
 
 // ─── external_files ──────────────────────────────────────────────────────────
 
-fn read_external_files(conn: &Connection) -> ParseResult<Option<HashMap<String, DucExternalFileData>>> {
+fn read_external_files(conn: &Connection) -> ParseResult<Option<HashMap<String, DucExternalFile>>> {
+    // Check if the new schema (with external_file_revisions) is present.
+    let has_revisions_table: bool = conn.prepare(
+        "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='external_file_revisions'"
+    )?.query_row([], |row| row.get::<_, i32>(0)).unwrap_or(0) > 0;
+
+    if has_revisions_table {
+        read_external_files_v2(conn)
+    } else {
+        read_external_files_v1_legacy(conn)
+    }
+}
+
+/// Read from the current schema (external_files + external_file_revisions).
+fn read_external_files_v2(conn: &Connection) -> ParseResult<Option<HashMap<String, DucExternalFile>>> {
+    // Load all revisions grouped by file_id.
+    let mut rev_stmt = conn.prepare(
+        "SELECT id, file_id, size_bytes, checksum, source_name, mime_type, message,
+                created, last_retrieved, data
+         FROM external_file_revisions"
+    )?;
+    let mut revisions_by_file: HashMap<String, HashMap<String, ExternalFileRevision>> = HashMap::new();
+    let rev_rows = rev_stmt.query_map([], |row| {
+        Ok(ExternalFileRevision {
+            id:             row.get(0)?,
+            size_bytes:     row.get(2)?,
+            checksum:       row.get(3)?,
+            source_name:    row.get(4)?,
+            mime_type:      row.get(5)?,
+            message:        row.get(6)?,
+            created:        row.get(7)?,
+            last_retrieved: row.get(8)?,
+            data:           row.get(9)?,
+        })
+    })?;
+
+    // We need file_id as well; re-query with it included.
+    let mut rev_stmt2 = conn.prepare(
+        "SELECT id, file_id, size_bytes, checksum, source_name, mime_type, message,
+                created, last_retrieved, data
+         FROM external_file_revisions"
+    )?;
+    // Suppress unused warning from first query
+    drop(rev_rows);
+
+    let rev_rows2 = rev_stmt2.query_map([], |row| {
+        let rev_id: String = row.get(0)?;
+        let file_id: String = row.get(1)?;
+        Ok((file_id, rev_id.clone(), ExternalFileRevision {
+            id:             rev_id,
+            size_bytes:     row.get(2)?,
+            checksum:       row.get(3)?,
+            source_name:    row.get(4)?,
+            mime_type:      row.get(5)?,
+            message:        row.get(6)?,
+            created:        row.get(7)?,
+            last_retrieved: row.get(8)?,
+            data:           row.get(9)?,
+        }))
+    })?;
+    for row in rev_rows2 {
+        let (file_id, rev_id, rev) = row?;
+        revisions_by_file.entry(file_id).or_default().insert(rev_id, rev);
+    }
+
+    // Load the file headers.
+    let mut file_stmt = conn.prepare(
+        "SELECT id, active_revision_id, updated, version FROM external_files"
+    )?;
+    let mut map: HashMap<String, DucExternalFile> = HashMap::new();
+    let file_rows = file_stmt.query_map([], |row| {
+        let id: String = row.get(0)?;
+        Ok((id.clone(), DucExternalFile {
+            id,
+            active_revision_id: row.get(1)?,
+            updated: row.get(2)?,
+            revisions: HashMap::new(),
+            version: row.get(3)?,
+        }))
+    })?;
+    for row in file_rows {
+        let (k, mut v) = row?;
+        v.revisions = revisions_by_file.remove(&k).unwrap_or_default();
+        map.insert(k, v);
+    }
+
+    if map.is_empty() { Ok(None) } else { Ok(Some(map)) }
+}
+
+/// Fallback for old schema files that only have the flat `external_files` table.
+/// Wraps each row in a single-revision DucExternalFile.
+fn read_external_files_v1_legacy(conn: &Connection) -> ParseResult<Option<HashMap<String, DucExternalFile>>> {
     let mut stmt = conn.prepare(
         "SELECT id, mime_type, data, created, last_retrieved, version FROM external_files"
     )?;
     let mut map = HashMap::new();
     let rows = stmt.query_map([], |row| {
         let id: String = row.get(0)?;
-        Ok((id.clone(), DucExternalFileData {
-            id,
-            mime_type: row.get(1)?,
-            data: row.get(2)?,
-            created: row.get(3)?,
-            last_retrieved: row.get(4)?,
+        let rev_id = format!("{}_rev1", id);
+        Ok((id.clone(), DucExternalFile {
+            id: id.clone(),
+            active_revision_id: rev_id.clone(),
+            updated: row.get(3)?,
             version: row.get(5)?,
+            revisions: {
+                let mut revs = HashMap::new();
+                revs.insert(rev_id.clone(), ExternalFileRevision {
+                    id: rev_id,
+                    size_bytes: 0,
+                    checksum: None,
+                    source_name: None,
+                    mime_type: row.get(1)?,
+                    message: None,
+                    created: row.get(3)?,
+                    last_retrieved: row.get(4)?,
+                    data: row.get(2)?,
+                });
+                revs
+            },
         }))
     })?;
     for row in rows {
         let (k, v) = row?;
         map.insert(k, v);
     }
-
     if map.is_empty() { Ok(None) } else { Ok(Some(map)) }
 }
 

{ducpy-3.0.4 → ducpy-3.1.0}/packages/ducrs/src/serialize.rs

@@ -1123,15 +1123,39 @@ fn write_hatch_pattern_lines(
 
 fn write_external_files(
     tx: &Transaction,
-    files: &Option<std::collections::HashMap<String, DucExternalFileData>>,
+    files: &Option<std::collections::HashMap<String, DucExternalFile>>,
 ) -> SerializeResult<()> {
     let Some(files) = files else { return Ok(()) };
-    let mut stmt = tx.prepare_cached(
-        "INSERT OR REPLACE INTO external_files (id, mime_type, data, created, last_retrieved, version)
-         VALUES (?1, ?2, ?3, ?4, ?5, ?6)"
+    let mut file_stmt = tx.prepare_cached(
+        "INSERT OR REPLACE INTO external_files (id, active_revision_id, updated, version)
+         VALUES (?1, ?2, ?3, ?4)"
+    )?;
+    let mut rev_stmt = tx.prepare_cached(
+        "INSERT OR REPLACE INTO external_file_revisions
+            (id, file_id, size_bytes, checksum, source_name, mime_type, message, created, last_retrieved, data)
+         VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)"
     )?;
-    for (_key, data) in files {
-        stmt.execute(params![data.id, data.mime_type, data.data, data.created, data.last_retrieved, data.version])?;
+    for (_key, file) in files {
+        file_stmt.execute(params![
+            file.id,
+            file.active_revision_id,
+            file.updated,
+            file.version,
+        ])?;
+        for (_rev_key, rev) in &file.revisions {
+            rev_stmt.execute(params![
+                rev.id,
+                file.id,
+                rev.size_bytes,
+                rev.checksum,
+                rev.source_name,
+                rev.mime_type,
+                rev.message,
+                rev.created,
+                rev.last_retrieved,
+                rev.data,
+            ])?;
+        }
     }
     Ok(())
 }

{ducpy-3.0.4 → ducpy-3.1.0}/packages/ducrs/src/types.rs

@@ -1,3 +1,5 @@
+use std::collections::HashMap;
+
 /**
  * Aligns text vertically within its bounding box.
  */
@@ -1461,40 +1463,51 @@ pub struct VersionGraph {
 
 // =============== EXTERNAL FILES ===============
 
-/// Lightweight metadata for an external file (no data blob).
+/// Snapshot of an external file at a given point in time.
 #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub struct ExternalFileMetadata {
+pub struct ExternalFileRevision {
+    // Revision number as string, functions as an incremental serial id
     pub id: String,
+    pub size_bytes: i64,
+    /// Content hash for integrity checks and optional deduplication.
+    pub checksum: Option<String>,
+    /// Original upload filename shown to the user.
+    pub source_name: Option<String>,
     pub mime_type: String,
+    /// Optional note describing what changed in this revision.
+    pub message: Option<String>,
+    /// Epoch ms when this revision was created.
     pub created: i64,
+    /// Epoch ms when this revision was last loaded onto the scene.
     pub last_retrieved: Option<i64>,
-    pub version: Option<i32>,
-}
 
+    /// The actual file content bytes
+    #[serde(with = "serde_bytes")]
+    pub data: Vec<u8>,
+}
 #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub struct DucExternalFileData {
-    pub mime_type: String,
+pub struct DucExternalFile {
     pub id: String,
-    /** The actual file content bytes. */
-    #[serde(with = "serde_bytes")]
-    pub data: Vec<u8>,
-    /** Epoch timestamp in milliseconds when the file was created. */
-    pub created: i64,
-    /**
-     * Epoch timestamp in milliseconds when the file was last retrieved from storage to be loaded onto the scene.
-     * Used to determine whether to delete unused files from storage.
-     */
-    pub last_retrieved: Option<i64>,
+    pub active_revision_id: String,
+    /// Epoch ms when the logical file was last mutated (revision added or active changed).
+    pub updated: i64,
+    // All revisions of this file, keyed by their `id`.
+    pub revisions: HashMap<String, ExternalFileRevision>,
     pub version: Option<i32>,
 }
 
+/// Lightweight summary of an external file used for lazy/metadata-only access
+/// (active revision's metadata + file-level version, no data blob).
 #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub struct DucExternalFileEntry {
-    pub key: String,
-    pub value: DucExternalFileData,
+pub struct ExternalFileMeta {
+    pub id: String,
+    pub mime_type: String,
+    pub created: i64,
+    pub last_retrieved: Option<i64>,
+    pub version: Option<i32>,
 }
 
 // =============== ROOT TYPE ===============
@@ -1509,7 +1522,7 @@ pub struct ExportedDataState {
     pub source: String,
     #[serde(rename = "type")]
     pub data_type: String,
-    pub dictionary: Option<std::collections::HashMap<String, String>>,
+    pub dictionary: Option<HashMap<String, String>>,
     #[serde(with = "serde_bytes", default, skip_serializing_if = "Option::is_none")]
     pub thumbnail: Option<Vec<u8>>,
     pub elements: Vec<ElementWrapper>,
@@ -1528,5 +1541,5 @@ pub struct ExportedDataState {
     /** In case it is needed to embed the version control into the file format */
     pub version_graph: Option<VersionGraph>,
     #[serde(rename = "files")]
-    pub external_files: Option<std::collections::HashMap<String, DucExternalFileData>>,
+    pub external_files: Option<HashMap<String, DucExternalFile>>,
 }