ducpy 3.1.1__tar.gz → 3.2.0__tar.gz

{ducpy-3.1.1 → ducpy-3.2.0}/Cargo.lock

@@ -411,7 +411,7 @@ dependencies = [
 
 [[package]]
 name = "ducpy-native"
-version = "3.1.1"
+version = "3.2.0"
 dependencies = [
  "duc",
  "pyo3",

{ducpy-3.1.1 → ducpy-3.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ducpy
-Version: 3.1.1
+Version: 3.2.0
 Classifier: License :: OSI Approved :: MIT License
 Requires-Dist: nanoid>=2.0.0
 License-File: LICENSE
@@ -19,7 +19,7 @@ Project-URL: Source, https://github.com/ducflair/duc/tree/main/packages/ducpy
 
 <p align="center">
   <br/>
-  <a href="https://duc.ducflair.com" target="_blank"><img width="256px" src="https://raw.githubusercontent.com/ducflair/assets/refs/heads/main/src/duc/duc-extended.png" /></a>
+  <a href="https://duc.ducflair.com" target="_blank"><img width="256px" src="https://cdn.jsdelivr.net/gh/ducflair/assets@main/src/duc/duc-extended.png" /></a>
   <p align="center">2D CAD File Format</p>
   <p align="center" style="align: center;">
     <a href="https://pypi.org/project/ducpy/"><img src="https://shields.io/badge/Pip-blue?logo=Pypi&logoColor=white&style=round-square" alt="Pip" /></a>

{ducpy-3.1.1 → ducpy-3.2.0}/README.md

@@ -2,7 +2,7 @@
 
 <p align="center">
   <br/>
-  <a href="https://duc.ducflair.com" target="_blank"><img width="256px" src="https://raw.githubusercontent.com/ducflair/assets/refs/heads/main/src/duc/duc-extended.png" /></a>
+  <a href="https://duc.ducflair.com" target="_blank"><img width="256px" src="https://cdn.jsdelivr.net/gh/ducflair/assets@main/src/duc/duc-extended.png" /></a>
   <p align="center">2D CAD File Format</p>
   <p align="center" style="align: center;">
     <a href="https://pypi.org/project/ducpy/"><img src="https://shields.io/badge/Pip-blue?logo=Pypi&logoColor=white&style=round-square" alt="Pip" /></a>

{ducpy-3.1.1 → ducpy-3.2.0}/packages/ducpy/crate/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ducpy-native"
-version = "3.1.1"
+version = "3.2.0"
 edition = "2021"
 description = "Python native extension for the duc file format (used by ducpy)"
 publish = false

{ducpy-3.1.1 → ducpy-3.2.0}/packages/ducrs/README.md

@@ -2,7 +2,7 @@
 
 <p align="center">
   <br/>
-  <a href="https://duc.ducflair.com" target="_blank"><img width="256px" src="https://raw.githubusercontent.com/ducflair/assets/refs/heads/main/src/duc/duc-extended.png" /></a>
+  <a href="https://duc.ducflair.com" target="_blank"><img width="256px" src="https://cdn.jsdelivr.net/gh/ducflair/assets@main/src/duc/duc-extended.png" /></a>
   <p align="center">2D CAD File Format</p>
   <p align="center" style="align: center;">
     <a href="https://crates.io/crates/duc/"><img src="https://shields.io/badge/Crates-FFC933?logo=Rust&logoColor=646464&style=round-square" alt="Crates" /></a>

{ducpy-3.1.1 → ducpy-3.2.0}/packages/ducrs/src/api/version_control.rs

@@ -313,7 +313,7 @@ impl<'a> VersionControl<'a> {
         self.conn.with(|c| {
             c.query_row(
                 "SELECT current_version, current_schema_version, chain_count,
-                        last_pruned, total_size
+                        total_size
                  FROM version_graph WHERE id = 1",
                 [],
                 |row| {
@@ -321,8 +321,7 @@ impl<'a> VersionControl<'a> {
                         current_version: row.get(0)?,
                         current_schema_version: row.get(1)?,
                         chain_count: row.get(2)?,
-                        last_pruned: row.get::<_, Option<i64>>(3)?.unwrap_or(0),
-                        total_size: row.get::<_, Option<i64>>(4)?.unwrap_or(0),
+                        total_size: row.get::<_, Option<i64>>(3)?.unwrap_or(0),
                     })
                 },
             )
@@ -465,93 +464,6 @@ impl<'a> VersionControl<'a> {
         })
     }
 
-    // ────────────────────────────────────────────────────────────────────────
-    // WRITE — Pruning
-    // ────────────────────────────────────────────────────────────────────────
-
-    /// Prune old deltas that precede the given `keep_after_version`.
-    /// Checkpoints are never pruned (they are self-contained recovery points).
-    /// Returns the number of deltas removed.
-    pub fn prune_deltas_before(&self, keep_after_version: i64) -> DbResult<usize> {
-        self.conn.with(|c| {
-            let removed = c
-                .execute(
-                    "DELETE FROM deltas WHERE version_number < ?1",
-                    [keep_after_version],
-                )
-                .map_err(DbError::from)?;
-
-            // Update last_pruned timestamp
-            let now_ms = std::time::SystemTime::now()
-                .duration_since(std::time::UNIX_EPOCH)
-                .unwrap_or_default()
-                .as_millis() as i64;
-
-            c.execute(
-                "UPDATE version_graph SET last_pruned = ?1 WHERE id = 1",
-                [now_ms],
-            )
-            .map_err(DbError::from)?;
-
-            // Recalculate total_size
-            self.recalculate_total_size(c)?;
-
-            Ok(removed)
-        })
-    }
-
-    /// Remove ALL versions (checkpoints + deltas) that are strictly older than
-    /// `keep_after_version`, except the most recent checkpoint before that
-    /// boundary (needed as a recovery base).
-    pub fn prune_before(&self, keep_after_version: i64) -> DbResult<usize> {
-        self.conn.with(|c| {
-            // Find the latest checkpoint at or before the boundary — keep it
-            let safe_cp_version: Option<i64> = c
-                .query_row(
-                    "SELECT MAX(version_number) FROM checkpoints
-                     WHERE version_number <= ?1",
-                    [keep_after_version],
-                    |row| row.get(0),
-                )
-                .optional()
-                .map_err(DbError::from)?
-                .flatten();
-
-            let safe_version = safe_cp_version.unwrap_or(0);
-
-            // Delete deltas before the safe checkpoint
-            let del_deltas = c
-                .execute(
-                    "DELETE FROM deltas WHERE version_number < ?1",
-                    [safe_version],
-                )
-                .map_err(DbError::from)?;
-
-            // Delete checkpoints before the safe checkpoint (keep the safe one)
-            let del_cps = c
-                .execute(
-                    "DELETE FROM checkpoints WHERE version_number < ?1",
-                    [safe_version],
-                )
-                .map_err(DbError::from)?;
-
-            let now_ms = std::time::SystemTime::now()
-                .duration_since(std::time::UNIX_EPOCH)
-                .unwrap_or_default()
-                .as_millis() as i64;
-
-            c.execute(
-                "UPDATE version_graph SET last_pruned = ?1 WHERE id = 1",
-                [now_ms],
-            )
-            .map_err(DbError::from)?;
-
-            self.recalculate_total_size(c)?;
-
-            Ok(del_deltas + del_cps)
-        })
-    }
-
     // ────────────────────────────────────────────────────────────────────────
     // WRITE — Revert
     // ────────────────────────────────────────────────────────────────────────
@@ -808,7 +720,7 @@ pub(crate) fn read_version_graph_inner(
     let mut vg_stmt = conn
         .prepare(
             "SELECT current_version, current_schema_version, user_checkpoint_version_id,
-                    latest_version_id, chain_count, last_pruned, total_size
+                    latest_version_id, chain_count, total_size
              FROM version_graph WHERE id = 1",
         )
         .map_err(DbError::from)?;
@@ -819,8 +731,7 @@ pub(crate) fn read_version_graph_inner(
                 current_version: row.get(0)?,
                 current_schema_version: row.get(1)?,
                 chain_count: row.get(4)?,
-                last_pruned: row.get::<_, Option<i64>>(5)?.unwrap_or(0),
-                total_size: row.get::<_, Option<i64>>(6)?.unwrap_or(0),
+                total_size: row.get::<_, Option<i64>>(5)?.unwrap_or(0),
             },
             row.get::<_, Option<String>>(2)?.unwrap_or_default(),
             row.get::<_, Option<String>>(3)?.unwrap_or_default(),

{ducpy-3.1.1 → ducpy-3.2.0}/packages/ducrs/src/lib.rs

@@ -114,6 +114,7 @@ mod tests {
                 .unwrap_or_else(|e| panic!("parse_lazy {} failed: {e}", path.display()));
 
             assert!(lazy.external_files.is_none(), "{}: lazy parse must omit external_files", path.display());
+            assert!(lazy.external_files_data.is_none(), "{}: lazy parse must omit external_files_data", path.display());
             assert_eq!(full.elements.len(), lazy.elements.len(), "{}: element count diverged", path.display());
             assert_eq!(full.layers.len(), lazy.layers.len(), "{}: layer count diverged", path.display());
         }
@@ -194,9 +195,9 @@ mod tests {
                 let file = parse::get_external_file(&buf, &meta.id)
                     .unwrap_or_else(|e| panic!("get_external_file {} id={} failed: {e}", path.display(), meta.id));
                 assert!(file.is_some(), "{}: listed file '{}' not found via get", path.display(), meta.id);
-                let file = file.unwrap();
-                assert_eq!(file.id, meta.id);
-                assert!(!file.revisions.is_empty(), "{}: file '{}' has no revisions", path.display(), meta.id);
+                let loaded = file.unwrap();
+                assert_eq!(loaded.file.id, meta.id);
+                assert!(!loaded.file.revisions.is_empty(), "{}: file '{}' has no revisions", path.display(), meta.id);
             }
         }
     }

{ducpy-3.1.1 → ducpy-3.2.0}/packages/ducrs/src/parse.rs

@@ -57,7 +57,7 @@ pub fn parse(buf: &[u8]) -> ParseResult<ExportedDataState> {
     let regions = read_regions(&conn)?;
     let (blocks, block_instances, block_collections) = read_blocks(&conn)?;
     let elements = read_elements(&conn)?;
-    let external_files = read_external_files(&conn)?;
+    let (external_files, external_files_data) = read_external_files(&conn)?;
     let version_graph = read_version_graph(&conn)?;
 
     Ok(ExportedDataState {
@@ -78,6 +78,7 @@ pub fn parse(buf: &[u8]) -> ParseResult<ExportedDataState> {
         duc_global_state,
         version_graph,
         external_files,
+        external_files_data,
     })
 }
 
@@ -117,16 +118,22 @@ pub fn parse_lazy(buf: &[u8]) -> ParseResult<ExportedDataState> {
         duc_global_state,
         version_graph,
         external_files: None,
+        external_files_data: None,
     })
 }
 
 /// Fetch a single external file from a `.duc` buffer by file ID.
-pub fn get_external_file(buf: &[u8], file_id: &str) -> ParseResult<Option<DucExternalFile>> {
+/// Returns metadata + data blobs as an `ExternalFileLoaded`.
+pub fn get_external_file(buf: &[u8], file_id: &str) -> ParseResult<Option<ExternalFileLoaded>> {
     let conn = load_db_bytes(buf)?;
     let has_revisions_table: bool = conn.prepare(
         "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='external_file_revisions'"
     )?.query_row([], |row| row.get::<_, i32>(0)).unwrap_or(0) > 0;
 
+    let has_data_table: bool = conn.prepare(
+        "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='external_file_revision_data'"
+    )?.query_row([], |row| row.get::<_, i32>(0)).unwrap_or(0) > 0;
+
     if has_revisions_table {
         // New schema: load file header + revisions.
         let file = conn.prepare(
@@ -146,30 +153,77 @@ pub fn get_external_file(buf: &[u8], file_id: &str) -> ParseResult<Option<DucExt
             Err(e) => return Err(e.into()),
         };
 
-        let mut rev_stmt = conn.prepare(
-            "SELECT id, size_bytes, checksum, source_name, mime_type, message,
-                    created, last_retrieved, data
-             FROM external_file_revisions WHERE file_id = ?1"
-        )?;
-        let rev_rows = rev_stmt.query_map(params![file_id], |row| {
-            let id: String = row.get(0)?;
-            Ok((id.clone(), ExternalFileRevision {
-                id,
-                size_bytes:     row.get(1)?,
-                checksum:       row.get(2)?,
-                source_name:    row.get(3)?,
-                mime_type:      row.get(4)?,
-                message:        row.get(5)?,
-                created:        row.get(6)?,
-                last_retrieved: row.get(7)?,
-                data:           row.get(8)?,
-            }))
-        })?;
-        for row in rev_rows {
-            let (rev_id, rev) = row?;
-            file.revisions.insert(rev_id, rev);
+        let mut data_map: HashMap<String, serde_bytes::ByteBuf> = HashMap::new();
+
+        if has_data_table {
+            // Split schema: metadata in external_file_revisions, data in external_file_revision_data
+            let mut meta_stmt = conn.prepare(
+                "SELECT id, size_bytes, checksum, source_name, mime_type, message,
+                        created, last_retrieved
+                 FROM external_file_revisions WHERE file_id = ?1"
+            )?;
+            let meta_rows = meta_stmt.query_map(params![file_id], |row| {
+                let id: String = row.get(0)?;
+                Ok((id.clone(), ExternalFileRevisionMeta {
+                    id,
+                    size_bytes:     row.get(1)?,
+                    checksum:       row.get(2)?,
+                    source_name:    row.get(3)?,
+                    mime_type:      row.get(4)?,
+                    message:        row.get(5)?,
+                    created:        row.get(6)?,
+                    last_retrieved: row.get(7)?,
+                }))
+            })?;
+            for row in meta_rows {
+                let (rev_id, meta) = row?;
+                file.revisions.insert(rev_id, meta);
+            }
+
+            let mut data_stmt = conn.prepare(
+                "SELECT d.revision_id, d.data
+                 FROM external_file_revision_data d
+                 JOIN external_file_revisions r ON r.id = d.revision_id
+                 WHERE r.file_id = ?1"
+            )?;
+            let data_rows = data_stmt.query_map(params![file_id], |row| {
+                let rev_id: String = row.get(0)?;
+                let blob: Vec<u8> = row.get(1)?;
+                Ok((rev_id, blob))
+            })?;
+            for row in data_rows {
+                let (rev_id, blob) = row?;
+                data_map.insert(rev_id, serde_bytes::ByteBuf::from(blob));
+            }
+        } else {
+            // Pre-split schema: data still in external_file_revisions
+            let mut rev_stmt = conn.prepare(
+                "SELECT id, size_bytes, checksum, source_name, mime_type, message,
+                        created, last_retrieved, data
+                 FROM external_file_revisions WHERE file_id = ?1"
+            )?;
+            let rev_rows = rev_stmt.query_map(params![file_id], |row| {
+                let id: String = row.get(0)?;
+                let data: Vec<u8> = row.get(8)?;
+                Ok((id.clone(), ExternalFileRevisionMeta {
+                    id,
+                    size_bytes:     row.get(1)?,
+                    checksum:       row.get(2)?,
+                    source_name:    row.get(3)?,
+                    mime_type:      row.get(4)?,
+                    message:        row.get(5)?,
+                    created:        row.get(6)?,
+                    last_retrieved: row.get(7)?,
+                }, data))
+            })?;
+            for row in rev_rows {
+                let (rev_id, meta, blob) = row?;
+                file.revisions.insert(rev_id.clone(), meta);
+                data_map.insert(rev_id, serde_bytes::ByteBuf::from(blob));
+            }
         }
-        Ok(Some(file))
+
+        Ok(Some(ExternalFileLoaded { file, data: data_map }))
     } else {
         // Legacy schema: wrap flat row into a single-revision DucExternalFile.
         let result = conn.prepare(
@@ -177,30 +231,34 @@ pub fn get_external_file(buf: &[u8], file_id: &str) -> ParseResult<Option<DucExt
         )?.query_row(params![file_id], |row| {
             let id: String = row.get(0)?;
             let rev_id = format!("{}_rev1", id);
-            Ok(DucExternalFile {
-                id: id.clone(),
-                active_revision_id: rev_id.clone(),
-                updated: row.get(3)?,
-                version: row.get(5)?,
-                revisions: {
-                    let mut revs = HashMap::new();
-                    revs.insert(rev_id.clone(), ExternalFileRevision {
-                        id: rev_id,
-                        size_bytes: 0,
-                        checksum: None,
-                        source_name: None,
-                        mime_type: row.get(1)?,
-                        message: None,
-                        created: row.get(3)?,
-                        last_retrieved: row.get(4)?,
-                        data: row.get(2)?,
-                    });
-                    revs
+            let blob: Vec<u8> = row.get(2)?;
+            let meta = ExternalFileRevisionMeta {
+                id: rev_id.clone(),
+                size_bytes: blob.len() as i64,
+                checksum: None,
+                source_name: None,
+                mime_type: row.get(1)?,
+                message: None,
+                created: row.get(3)?,
+                last_retrieved: row.get(4)?,
+            };
+            let mut revisions = HashMap::new();
+            revisions.insert(rev_id.clone(), meta);
+            let mut data_map = HashMap::new();
+            data_map.insert(rev_id.clone(), serde_bytes::ByteBuf::from(blob));
+            Ok(ExternalFileLoaded {
+                file: DucExternalFile {
+                    id: id.clone(),
+                    active_revision_id: rev_id,
+                    updated: row.get(3)?,
+                    version: row.get(5)?,
+                    revisions,
                 },
+                data: data_map,
             })
         });
         match result {
-            Ok(file) => Ok(Some(file)),
+            Ok(loaded) => Ok(Some(loaded)),
             Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
             Err(e) => Err(e.into()),
         }
@@ -579,14 +637,6 @@ fn int_to_image_status(v: i32) -> IMAGE_STATUS {
         _ => IMAGE_STATUS::PENDING,
     }
 }
-fn int_to_pruning_level(v: i32) -> PRUNING_LEVEL {
-    match v {
-        10 => PRUNING_LEVEL::CONSERVATIVE,
-        20 => PRUNING_LEVEL::BALANCED,
-        30 => PRUNING_LEVEL::AGGRESSIVE,
-        _ => PRUNING_LEVEL::BALANCED,
-    }
-}
 fn int_to_boolean_operation(v: i32) -> BOOLEAN_OPERATION {
     match v {
         10 => BOOLEAN_OPERATION::UNION,
@@ -636,7 +686,7 @@ fn read_document(conn: &Connection) -> ParseResult<(Option<String>, String, Stri
 
 fn read_global_state(conn: &Connection) -> ParseResult<Option<DucGlobalState>> {
     let mut stmt = conn.prepare(
-        "SELECT name, view_background_color, main_scope, scope_exponent_threshold, pruning_level
+        "SELECT name, view_background_color, main_scope, scope_exponent_threshold
          FROM duc_global_state WHERE id = 1"
     )?;
     let result = stmt.query_row([], |row| {
@@ -645,7 +695,6 @@ fn read_global_state(conn: &Connection) -> ParseResult<Option<DucGlobalState>> {
             view_background_color: row.get(1)?,
             main_scope: row.get(2)?,
             scope_exponent_threshold: row.get(3)?,
-            pruning_level: int_to_pruning_level(row.get(4)?),
         })
     });
 
@@ -1768,31 +1817,46 @@ fn read_hatch_pattern_lines(conn: &Connection, owner_type: &str, owner_id: i64)
 
 // ─── external_files ──────────────────────────────────────────────────────────
 
-fn read_external_files(conn: &Connection) -> ParseResult<Option<HashMap<String, DucExternalFile>>> {
+type ExternalFilesResult = (
+    Option<HashMap<String, DucExternalFile>>,
+    Option<HashMap<String, serde_bytes::ByteBuf>>,
+);
+
+fn read_external_files(conn: &Connection) -> ParseResult<ExternalFilesResult> {
     // Check if the new schema (with external_file_revisions) is present.
     let has_revisions_table: bool = conn.prepare(
         "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='external_file_revisions'"
     )?.query_row([], |row| row.get::<_, i32>(0)).unwrap_or(0) > 0;
 
     if has_revisions_table {
-        read_external_files_v2(conn)
+        let has_data_table: bool = conn.prepare(
+            "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='external_file_revision_data'"
+        )?.query_row([], |row| row.get::<_, i32>(0)).unwrap_or(0) > 0;
+
+        if has_data_table {
+            read_external_files_v3(conn)
+        } else {
+            read_external_files_v2(conn)
+        }
     } else {
         read_external_files_v1_legacy(conn)
     }
 }
 
-/// Read from the current schema (external_files + external_file_revisions).
-fn read_external_files_v2(conn: &Connection) -> ParseResult<Option<HashMap<String, DucExternalFile>>> {
-    // Load all revisions grouped by file_id.
+/// Read from the split schema (external_file_revisions metadata + external_file_revision_data).
+fn read_external_files_v3(conn: &Connection) -> ParseResult<ExternalFilesResult> {
+    // Load revision metadata
     let mut rev_stmt = conn.prepare(
         "SELECT id, file_id, size_bytes, checksum, source_name, mime_type, message,
-                created, last_retrieved, data
+                created, last_retrieved
          FROM external_file_revisions"
     )?;
-    let mut revisions_by_file: HashMap<String, HashMap<String, ExternalFileRevision>> = HashMap::new();
+    let mut revisions_by_file: HashMap<String, HashMap<String, ExternalFileRevisionMeta>> = HashMap::new();
     let rev_rows = rev_stmt.query_map([], |row| {
-        Ok(ExternalFileRevision {
-            id:             row.get(0)?,
+        let rev_id: String = row.get(0)?;
+        let file_id: String = row.get(1)?;
+        Ok((file_id, rev_id.clone(), ExternalFileRevisionMeta {
+            id:             rev_id,
             size_bytes:     row.get(2)?,
             checksum:       row.get(3)?,
             source_name:    row.get(4)?,
@@ -1800,23 +1864,69 @@ fn read_external_files_v2(conn: &Connection) -> ParseResult<Option<HashMap<Strin
             message:        row.get(6)?,
             created:        row.get(7)?,
             last_retrieved: row.get(8)?,
-            data:           row.get(9)?,
-        })
+        }))
+    })?;
+    for row in rev_rows {
+        let (file_id, rev_id, meta) = row?;
+        revisions_by_file.entry(file_id).or_default().insert(rev_id, meta);
+    }
+
+    // Load data blobs
+    let mut data_stmt = conn.prepare(
+        "SELECT revision_id, data FROM external_file_revision_data"
+    )?;
+    let mut data_map: HashMap<String, serde_bytes::ByteBuf> = HashMap::new();
+    let data_rows = data_stmt.query_map([], |row| {
+        let rev_id: String = row.get(0)?;
+        let blob: Vec<u8> = row.get(1)?;
+        Ok((rev_id, blob))
     })?;
+    for row in data_rows {
+        let (rev_id, blob) = row?;
+        data_map.insert(rev_id, serde_bytes::ByteBuf::from(blob));
+    }
+
+    // Load file headers
+    let mut file_stmt = conn.prepare(
+        "SELECT id, active_revision_id, updated, version FROM external_files"
+    )?;
+    let mut map: HashMap<String, DucExternalFile> = HashMap::new();
+    let file_rows = file_stmt.query_map([], |row| {
+        let id: String = row.get(0)?;
+        Ok((id.clone(), DucExternalFile {
+            id,
+            active_revision_id: row.get(1)?,
+            updated: row.get(2)?,
+            revisions: HashMap::new(),
+            version: row.get(3)?,
+        }))
+    })?;
+    for row in file_rows {
+        let (k, mut v) = row?;
+        v.revisions = revisions_by_file.remove(&k).unwrap_or_default();
+        map.insert(k, v);
+    }
 
-    // We need file_id as well; re-query with it included.
-    let mut rev_stmt2 = conn.prepare(
+    let files = if map.is_empty() { None } else { Some(map) };
+    let data = if data_map.is_empty() { None } else { Some(data_map) };
+    Ok((files, data))
+}
+
+/// Read from the pre-split schema (external_files + external_file_revisions with data column).
+fn read_external_files_v2(conn: &Connection) -> ParseResult<ExternalFilesResult> {
+    let mut rev_stmt = conn.prepare(
         "SELECT id, file_id, size_bytes, checksum, source_name, mime_type, message,
                 created, last_retrieved, data
          FROM external_file_revisions"
     )?;
-    // Suppress unused warning from first query
-    drop(rev_rows);
+    let mut revisions_by_file: HashMap<String, HashMap<String, ExternalFileRevisionMeta>> = HashMap::new();
+    let mut data_map: HashMap<String, serde_bytes::ByteBuf> = HashMap::new();
 
-    let rev_rows2 = rev_stmt2.query_map([], |row| {
+    let rev_rows = rev_stmt.query_map([], |row| {
         let rev_id: String = row.get(0)?;
         let file_id: String = row.get(1)?;
-        Ok((file_id, rev_id.clone(), ExternalFileRevision {
+        let blob: Vec<u8> = row.get(9)?;
+        Ok((file_id, rev_id.clone(), ExternalFileRevisionMeta {
             id:             rev_id,
             size_bytes:     row.get(2)?,
             checksum:       row.get(3)?,
@@ -1825,12 +1935,12 @@ fn read_external_files_v2(conn: &Connection) -> ParseResult<Option<HashMap<Strin
             message:        row.get(6)?,
             created:        row.get(7)?,
             last_retrieved: row.get(8)?,
-            data:           row.get(9)?,
-        }))
+        }, blob))
     })?;
-    for row in rev_rows2 {
-        let (file_id, rev_id, rev) = row?;
-        revisions_by_file.entry(file_id).or_default().insert(rev_id, rev);
+    for row in rev_rows {
+        let (file_id, rev_id, meta, blob) = row?;
+        revisions_by_file.entry(file_id).or_default().insert(rev_id.clone(), meta);
+        data_map.insert(rev_id, serde_bytes::ByteBuf::from(blob));
     }
 
     // Load the file headers.
@@ -1854,46 +1964,53 @@ fn read_external_files_v2(conn: &Connection) -> ParseResult<Option<HashMap<Strin
         map.insert(k, v);
     }
 
-    if map.is_empty() { Ok(None) } else { Ok(Some(map)) }
+    let files = if map.is_empty() { None } else { Some(map) };
+    let data = if data_map.is_empty() { None } else { Some(data_map) };
+    Ok((files, data))
 }
 
 /// Fallback for old schema files that only have the flat `external_files` table.
 /// Wraps each row in a single-revision DucExternalFile.
-fn read_external_files_v1_legacy(conn: &Connection) -> ParseResult<Option<HashMap<String, DucExternalFile>>> {
+fn read_external_files_v1_legacy(conn: &Connection) -> ParseResult<ExternalFilesResult> {
     let mut stmt = conn.prepare(
         "SELECT id, mime_type, data, created, last_retrieved, version FROM external_files"
     )?;
     let mut map = HashMap::new();
+    let mut data_map: HashMap<String, serde_bytes::ByteBuf> = HashMap::new();
     let rows = stmt.query_map([], |row| {
         let id: String = row.get(0)?;
         let rev_id = format!("{}_rev1", id);
-        Ok((id.clone(), DucExternalFile {
-            id: id.clone(),
+        let blob: Vec<u8> = row.get(2)?;
+        let meta = ExternalFileRevisionMeta {
+            id: rev_id.clone(),
+            size_bytes: blob.len() as i64,
+            checksum: None,
+            source_name: None,
+            mime_type: row.get(1)?,
+            message: None,
+            created: row.get(3)?,
+            last_retrieved: row.get(4)?,
+        };
+        Ok((id.clone(), rev_id.clone(), DucExternalFile {
+            id,
             active_revision_id: rev_id.clone(),
             updated: row.get(3)?,
             version: row.get(5)?,
             revisions: {
                 let mut revs = HashMap::new();
-                revs.insert(rev_id.clone(), ExternalFileRevision {
-                    id: rev_id,
-                    size_bytes: 0,
-                    checksum: None,
-                    source_name: None,
-                    mime_type: row.get(1)?,
-                    message: None,
-                    created: row.get(3)?,
-                    last_retrieved: row.get(4)?,
-                    data: row.get(2)?,
-                });
+                revs.insert(rev_id.clone(), meta);
                 revs
             },
-        }))
+        }, blob))
     })?;
     for row in rows {
-        let (k, v) = row?;
+        let (k, rev_id, v, blob) = row?;
+        data_map.insert(rev_id, serde_bytes::ByteBuf::from(blob));
         map.insert(k, v);
     }
-    if map.is_empty() { Ok(None) } else { Ok(Some(map)) }
+    let files = if map.is_empty() { None } else { Some(map) };
+    let data = if data_map.is_empty() { None } else { Some(data_map) };
+    Ok((files, data))
 }
 
 // ─── version_graph ───────────────────────────────────────────────────────────
@@ -1910,7 +2027,7 @@ fn read_version_graph(conn: &Connection) -> ParseResult<Option<VersionGraph>> {
 
     let mut vg_stmt = conn.prepare(
         "SELECT current_version, current_schema_version, user_checkpoint_version_id,
-                latest_version_id, chain_count, last_pruned, total_size
+                latest_version_id, chain_count, total_size
          FROM version_graph WHERE id = 1"
     )?;
     let (metadata, user_cp_id, latest_id) = match vg_stmt.query_row([], |row| {
@@ -1919,8 +2036,7 @@ fn read_version_graph(conn: &Connection) -> ParseResult<Option<VersionGraph>> {
                 current_version: row.get(0)?,
                 current_schema_version: row.get(1)?,
                 chain_count: row.get(4)?,
-                last_pruned: row.get::<_, Option<i64>>(5)?.unwrap_or(0),
-                total_size: row.get::<_, Option<i64>>(6)?.unwrap_or(0),
+                total_size: row.get::<_, Option<i64>>(5)?.unwrap_or(0),
             },
             row.get::<_, Option<String>>(2)?.unwrap_or_default(),
             row.get::<_, Option<String>>(3)?.unwrap_or_default(),

{ducpy-3.1.1 → ducpy-3.2.0}/packages/ducrs/src/serialize.rs

@@ -68,7 +68,7 @@ pub fn serialize(state: &ExportedDataState) -> SerializeResult<Vec<u8>> {
         write_stack_and_containers(&tx, state)?;
         write_blocks(&tx, state)?;
         write_elements(&tx, state)?;
-        write_external_files(&tx, &state.external_files)?;
+        write_external_files(&tx, &state.external_files, &state.external_files_data)?;
         write_version_graph(&tx, &state.version_graph)?;
         tx.commit()?;
 
@@ -149,14 +149,13 @@ fn write_global_state(tx: &Transaction, gs: &Option<DucGlobalState>) -> Serializ
     let Some(gs) = gs else { return Ok(()) };
     tx.execute(
         "INSERT OR REPLACE INTO duc_global_state
-            (id, name, view_background_color, main_scope, scope_exponent_threshold, pruning_level)
-         VALUES (1, ?1, ?2, ?3, ?4, ?5)",
+            (id, name, view_background_color, main_scope, scope_exponent_threshold)
+         VALUES (1, ?1, ?2, ?3, ?4)",
         params![
             gs.name,
             gs.view_background_color,
             gs.main_scope,
             gs.scope_exponent_threshold,
-            gs.pruning_level as i32,
         ],
     )?;
     Ok(())
@@ -1124,6 +1123,7 @@ fn write_hatch_pattern_lines(
 fn write_external_files(
     tx: &Transaction,
     files: &Option<std::collections::HashMap<String, DucExternalFile>>,
+    files_data: &Option<std::collections::HashMap<String, serde_bytes::ByteBuf>>,
 ) -> SerializeResult<()> {
     let Some(files) = files else { return Ok(()) };
     let mut file_stmt = tx.prepare_cached(
@@ -1132,8 +1132,12 @@ fn write_external_files(
     )?;
     let mut rev_stmt = tx.prepare_cached(
         "INSERT OR REPLACE INTO external_file_revisions
-            (id, file_id, size_bytes, checksum, source_name, mime_type, message, created, last_retrieved, data)
-         VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)"
+            (id, file_id, size_bytes, checksum, source_name, mime_type, message, created, last_retrieved)
+         VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)"
+    )?;
+    let mut data_stmt = tx.prepare_cached(
+        "INSERT OR REPLACE INTO external_file_revision_data (revision_id, data)
+         VALUES (?1, ?2)"
     )?;
     for (_key, file) in files {
         file_stmt.execute(params![
@@ -1153,10 +1157,15 @@ fn write_external_files(
                 rev.message,
                 rev.created,
                 rev.last_retrieved,
-                rev.data,
             ])?;
         }
     }
+    // Write data blobs from the separate filesData map
+    if let Some(data_map) = files_data {
+        for (rev_id, blob) in data_map {
+            data_stmt.execute(params![rev_id, blob.as_ref()])?;
+        }
+    }
     Ok(())
 }
 
@@ -1172,8 +1181,7 @@ fn write_version_graph(tx: &Transaction, vg: &Option<VersionGraph>) -> Serialize
             user_checkpoint_version_id = ?3,
             latest_version_id = ?4,
             chain_count = ?5,
-            last_pruned = ?6,
-            total_size = ?7
+            total_size = ?6
          WHERE id = 1",
         params![
             vg.metadata.current_version,
@@ -1181,7 +1189,6 @@ fn write_version_graph(tx: &Transaction, vg: &Option<VersionGraph>) -> Serialize
             vg.user_checkpoint_version_id,
             vg.latest_version_id,
             vg.metadata.chain_count,
-            vg.metadata.last_pruned,
             vg.metadata.total_size,
         ],
     )?;

{ducpy-3.1.1 → ducpy-3.2.0}/packages/ducrs/src/types.rs

@@ -249,18 +249,6 @@ pub enum IMAGE_STATUS {
     ERROR = 12,
 }
 
-#[allow(non_camel_case_types)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde_repr::Serialize_repr, serde_repr::Deserialize_repr)]
-#[repr(i32)]
-pub enum PRUNING_LEVEL {
-    /** Conservative pruning, retains more history. */
-    CONSERVATIVE = 10,
-    /** Balanced pruning, optimizes between history and size. */
-    BALANCED = 20,
-    /** Aggressive pruning, retains less history for smaller size. */
-    AGGRESSIVE = 30,
-}
-
 /**
  * Defines the types of boolean operations that can be performed.
  */
@@ -1326,8 +1314,6 @@ pub struct DucGlobalState {
      * This value defines a +/- tolerance range around the exponent of the current scope.
      */
     pub scope_exponent_threshold: i8,
-    /** The level of pruning to the versions from the version graph. */
-    pub pruning_level: PRUNING_LEVEL,
 }
 
 #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
@@ -1464,7 +1450,6 @@ pub struct VersionGraphMetadata {
     pub current_schema_version: i32,
     #[serde(deserialize_with = "crate::serde_utils::trunc_i32")]
     pub chain_count: i32,
-    pub last_pruned: i64,
     pub total_size: i64,
 }
 
@@ -1485,11 +1470,11 @@ pub struct VersionGraph {
 
 // =============== EXTERNAL FILES ===============
 
-/// Snapshot of an external file at a given point in time.
+/// Revision metadata without the heavy data blob.
+/// Used inside `DucExternalFile.revisions` so the file record stays lightweight.
 #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub struct ExternalFileRevision {
-    // Revision number as string, functions as an incremental serial id
+pub struct ExternalFileRevisionMeta {
     pub id: String,
     pub size_bytes: i64,
     /// Content hash for integrity checks and optional deduplication.
@@ -1503,11 +1488,8 @@ pub struct ExternalFileRevision {
     pub created: i64,
     /// Epoch ms when this revision was last loaded onto the scene.
     pub last_retrieved: Option<i64>,
-
-    /// The actual file content bytes
-    #[serde(with = "serde_bytes")]
-    pub data: Vec<u8>,
 }
+
 #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct DucExternalFile {
@@ -1515,11 +1497,21 @@ pub struct DucExternalFile {
     pub active_revision_id: String,
     /// Epoch ms when the logical file was last mutated (revision added or active changed).
     pub updated: i64,
-    // All revisions of this file, keyed by their `id`.
-    pub revisions: HashMap<String, ExternalFileRevision>,
+    /// All revisions of this file (metadata only, no data blobs).
+    pub revisions: HashMap<String, ExternalFileRevisionMeta>,
     pub version: Option<i32>,
 }
 
+/// Result of on-demand file loading — includes both metadata and data blobs.
+#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ExternalFileLoaded {
+    #[serde(flatten)]
+    pub file: DucExternalFile,
+    /// Revision data blobs keyed by revision id.
+    pub data: HashMap<String, serde_bytes::ByteBuf>,
+}
+
 /// Lightweight summary of an external file used for lazy/metadata-only access
 /// (active revision's metadata + file-level version, no data blob).
 #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
@@ -1564,4 +1556,8 @@ pub struct ExportedDataState {
     pub version_graph: Option<VersionGraph>,
     #[serde(rename = "files")]
     pub external_files: Option<HashMap<String, DucExternalFile>>,
+    /// Binary data blobs for external file revisions, keyed by revision id.
+    /// Separated from `external_files` so metadata can travel without heavy blobs.
+    #[serde(rename = "filesData", default, skip_serializing_if = "Option::is_none")]
+    pub external_files_data: Option<HashMap<String, serde_bytes::ByteBuf>>,
 }