ducpy 3.6.1__tar.gz → 4.0.0__tar.gz

{ducpy-3.6.1 → ducpy-4.0.0}/Cargo.lock

@@ -403,16 +403,20 @@ name = "ducjs-wasm"
 version = "0.0.0-development"
 dependencies = [
  "duc",
+ "flate2",
  "js-sys",
+ "rusqlite",
  "serde",
  "serde-wasm-bindgen",
  "serde_json",
+ "sqlite-wasm-vfs",
  "wasm-bindgen",
+ "wasm-bindgen-futures",
 ]
 
 [[package]]
 name = "ducpy-native"
-version = "3.6.1"
+version = "4.0.0"
 dependencies = [
  "duc",
  "pyo3",
@@ -1127,6 +1131,16 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "minicov"
+version = "0.3.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4869b6a491569605d66d3952bcdf03df789e5b536e5f0cf7758a7f08a55ae24d"
+dependencies = [
+ "cc",
+ "walkdir",
+]
+
 [[package]]
 name = "miniz_oxide"
 version = "0.8.9"
@@ -1587,9 +1601,9 @@ checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97"
 
 [[package]]
 name = "rsqlite-vfs"
-version = "0.1.0"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8a1f2315036ef6b1fbacd1972e8ee7688030b0a2121edfc2a6550febd41574d"
+checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c"
 dependencies = [
  "hashbrown",
  "thiserror",
@@ -1655,6 +1669,15 @@ version = "1.0.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
 
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
 [[package]]
 name = "sealed"
 version = "0.6.0"
@@ -1831,16 +1854,16 @@ dependencies = [
 [[package]]
 name = "sqlite-wasm-vfs"
 version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f7a5c9ac229421d577bb5a9bb59048838509958b218dd4e0b3c1214a87c361e"
 dependencies = [
  "indexed_db_futures",
  "js-sys",
  "rsqlite-vfs",
+ "sqlite-wasm-rs",
  "thiserror",
  "tokio",
  "wasm-bindgen",
  "wasm-bindgen-futures",
+ "wasm-bindgen-test",
  "web-sys",
 ]
 
@@ -2183,6 +2206,16 @@ version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
 
+[[package]]
+name = "walkdir"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
 [[package]]
 name = "wasip2"
 version = "1.0.2+wasi-0.2.9"
@@ -2264,6 +2297,30 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "wasm-bindgen-test"
+version = "0.3.54"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e381134e148c1062f965a42ed1f5ee933eef2927c3f70d1812158f711d39865"
+dependencies = [
+ "js-sys",
+ "minicov",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "wasm-bindgen-test-macro",
+]
+
+[[package]]
+name = "wasm-bindgen-test-macro"
+version = "0.3.54"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b673bca3298fe582aeef8352330ecbad91849f85090805582400850f8270a2e8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "web-sys"
 version = "0.3.81"

{ducpy-3.6.1 → ducpy-4.0.0}/Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
 resolver = "2"
-members = ["packages/ducrs", "packages/ducpy/crate"]
+members = ["packages/ducrs", "packages/ducpy/crate", "patches/sqlite-wasm-vfs"]
 
 [workspace.metadata.wasm-pack]
 "wasm-pack" = "0.14.0"
@@ -18,3 +18,6 @@ opt-level = "z"
 codegen-units = 1
 strip = true
 panic = "abort"
+
+[patch.crates-io]
+sqlite-wasm-vfs = { path = "patches/sqlite-wasm-vfs" }

{ducpy-3.6.1 → ducpy-4.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ducpy
-Version: 3.6.1
+Version: 4.0.0
 Classifier: License :: OSI Approved :: MIT License
 Requires-Dist: nanoid>=2.0.0
 Requires-Dist: pypdf>=4.2.0
@@ -85,7 +85,7 @@ Read and write `.duc` files using the `duc.parse` and `duc.serialize` modules.
 
 See the worked examples:
 
-- [Parsing](src/examples/parsing_demo.py) — `parse_duc` / `parse_duc_lazy` / `get_external_file` / `list_external_files`.
+- [Parsing](src/examples/parsing_demo.py) — `parse_duc` / `list_external_files` / streaming revision helpers.
 - [Serialization](src/examples/serialization_demo.py) — `serialize_duc` from builder-created elements.
 - [External files](src/examples/external_files_demo.py) — attaching binary blobs (images, PDFs) to a `.duc` document.
 
@@ -108,3 +108,4 @@ Feel free to open an issue or contact us at [Ducflair Support](https://www.ducfl
 ## License
 
 This project is licensed under the MIT License. See the [LICENSE](./LICENSE) file for more details.
+

{ducpy-3.6.1 → ducpy-4.0.0}/README.md

@@ -62,7 +62,7 @@ Read and write `.duc` files using the `duc.parse` and `duc.serialize` modules.
 
 See the worked examples:
 
-- [Parsing](src/examples/parsing_demo.py) — `parse_duc` / `parse_duc_lazy` / `get_external_file` / `list_external_files`.
+- [Parsing](src/examples/parsing_demo.py) — `parse_duc` / `list_external_files` / streaming revision helpers.
 - [Serialization](src/examples/serialization_demo.py) — `serialize_duc` from builder-created elements.
 - [External files](src/examples/external_files_demo.py) — attaching binary blobs (images, PDFs) to a `.duc` document.
 
@@ -84,4 +84,4 @@ Feel free to open an issue or contact us at [Ducflair Support](https://www.ducfl
 
 ## License
 
-This project is licensed under the MIT License. See the [LICENSE](./LICENSE) file for more details.
+This project is licensed under the MIT License. See the [LICENSE](./LICENSE) file for more details.

{ducpy-3.6.1 → ducpy-4.0.0}/packages/ducpy/crate/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ducpy-native"
-version = "3.6.1"
+version = "4.0.0"
 edition = "2021"
 description = "Python native extension for the duc file format (used by ducpy)"
 publish = false

{ducpy-3.6.1 → ducpy-4.0.0}/packages/ducpy/crate/src/lib.rs

@@ -1,60 +1,85 @@
 use pyo3::prelude::*;
-use pyo3::types::PyBytes;
+use std::fs::File;
 
-/// Parse a `.duc` file (bytes) into a Python dict (ExportedDataState).
+/// Parse a `.duc` file path into a Python dict (ExportedDataState).
 #[pyfunction]
-fn parse_duc(py: Python<'_>, buf: &[u8]) -> PyResult<PyObject> {
-    let state = duc::parse::parse(buf)
+fn parse_duc(py: Python<'_>, path: &str) -> PyResult<PyObject> {
+    let session = duc::session::DucSession::open_path(path)
+        .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("{e}")))?;
+    let state = session
+        .read_document_state()
         .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("{e}")))?;
     pythonize::pythonize(py, &state)
         .map(|b| b.unbind())
         .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("{e}")))
 }
 
-/// Parse a `.duc` file lazily — returns everything EXCEPT external file data blobs.
-///
-/// Use `get_external_file()` or `list_external_files()` for on-demand access.
+/// Serialize a Python dict (ExportedDataState) directly to a `.duc` output path.
 #[pyfunction]
-fn parse_duc_lazy(py: Python<'_>, buf: &[u8]) -> PyResult<PyObject> {
-    let state = duc::parse::parse_lazy(buf)
+fn serialize_duc(data: &Bound<'_, pyo3::types::PyAny>, output_path: &str) -> PyResult<()> {
+    let state: duc::types::ExportedDataState = pythonize::depythonize(data)
         .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("{e}")))?;
-    pythonize::pythonize(py, &state)
-        .map(|b| b.unbind())
+    let mut session = duc::session::DucSession::create_export_session()
+        .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("{e}")))?;
+    session
+        .write_document_state(&state)
+        .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("{e}")))?;
+    session
+        .finish_to_path(output_path)
         .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("{e}")))
 }
 
-/// Serialize a Python dict (ExportedDataState) into `.duc` bytes.
+/// List metadata for all external files without loading data blobs.
 #[pyfunction]
-fn serialize_duc(py: Python<'_>, data: &Bound<'_, pyo3::types::PyAny>) -> PyResult<PyObject> {
-    let state: duc::types::ExportedDataState = pythonize::depythonize(data)
+fn list_external_files(py: Python<'_>, path: &str) -> PyResult<PyObject> {
+    let session = duc::session::DucSession::open_path(path)
         .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("{e}")))?;
-    let bytes = duc::serialize::serialize(&state)
+    let meta = session
+        .list_external_files()
+        .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("{e}")))?;
+    pythonize::pythonize(py, &meta)
+        .map(|b| b.unbind())
+        .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("{e}")))
+}
+
+#[pyfunction]
+fn stream_external_file_revision_to_path(
+    path: &str,
+    revision_id: &str,
+    output_path: &str,
+) -> PyResult<u64> {
+    let session = duc::session::DucSession::open_path(path)
         .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("{e}")))?;
-    Ok(PyBytes::new(py, &bytes).into())
+    let mut out = File::create(output_path)
+        .map_err(|e| pyo3::exceptions::PyOSError::new_err(format!("{e}")))?;
+    session
+        .stream_external_file_revision_to_writer(revision_id, &mut out)
+        .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("{e}")))
 }
 
-/// Fetch a single external file from a `.duc` buffer by file ID.
-///
-/// Returns the file entry as a dict, or None if not found.
 #[pyfunction]
-fn get_external_file(py: Python<'_>, buf: &[u8], file_id: &str) -> PyResult<PyObject> {
-    let entry = duc::parse::get_external_file(buf, file_id)
+fn stream_checkpoint_data_to_path(
+    path: &str,
+    checkpoint_id: &str,
+    output_path: &str,
+) -> PyResult<u64> {
+    let session = duc::session::DucSession::open_path(path)
         .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("{e}")))?;
-    match entry {
-        Some(e) => pythonize::pythonize(py, &e)
-            .map(|b| b.unbind())
-            .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("{e}"))),
-        None => Ok(py.None()),
-    }
+    let mut out = File::create(output_path)
+        .map_err(|e| pyo3::exceptions::PyOSError::new_err(format!("{e}")))?;
+    session
+        .stream_checkpoint_data_to_writer(checkpoint_id, &mut out)
+        .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("{e}")))
 }
 
-/// List metadata for all external files (without loading the heavy data blobs).
 #[pyfunction]
-fn list_external_files(py: Python<'_>, buf: &[u8]) -> PyResult<PyObject> {
-    let meta = duc::parse::list_external_files(buf)
+fn stream_delta_changeset_to_path(path: &str, delta_id: &str, output_path: &str) -> PyResult<u64> {
+    let session = duc::session::DucSession::open_path(path)
         .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("{e}")))?;
-    pythonize::pythonize(py, &meta)
-        .map(|b| b.unbind())
+    let mut out = File::create(output_path)
+        .map_err(|e| pyo3::exceptions::PyOSError::new_err(format!("{e}")))?;
+    session
+        .stream_delta_changeset_to_writer(delta_id, &mut out)
         .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("{e}")))
 }
 
@@ -101,10 +126,11 @@ fn get_migrations(_py: Python<'_>) -> PyResult<Vec<(i64, i64, String)>> {
 #[pymodule]
 fn ducpy_native(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(parse_duc, m)?)?;
-    m.add_function(wrap_pyfunction!(parse_duc_lazy, m)?)?;
     m.add_function(wrap_pyfunction!(serialize_duc, m)?)?;
-    m.add_function(wrap_pyfunction!(get_external_file, m)?)?;
     m.add_function(wrap_pyfunction!(list_external_files, m)?)?;
+    m.add_function(wrap_pyfunction!(stream_external_file_revision_to_path, m)?)?;
+    m.add_function(wrap_pyfunction!(stream_checkpoint_data_to_path, m)?)?;
+    m.add_function(wrap_pyfunction!(stream_delta_changeset_to_path, m)?)?;
     m.add_function(wrap_pyfunction!(get_schema_version, m)?)?;
     m.add_function(wrap_pyfunction!(get_schema_version_int, m)?)?;
     m.add_function(wrap_pyfunction!(get_duc_schema_sql, m)?)?;

{ducpy-3.6.1 → ducpy-4.0.0}/packages/ducrs/Cargo.toml

@@ -28,11 +28,11 @@ serde_json = "1.0"
 serde_repr = "0.1"
 serde_bytes = "0.11"
 lazy_static = "1.4"
-rusqlite = { version = "0.38", features = ["bundled"] }
+rusqlite = { version = "0.38", features = ["bundled", "serialize"] }
 thiserror = "2"
 
 [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies]
-sqlite-wasm-vfs = { version = "0.2", optional = true }
+sqlite-wasm-vfs = { path = "../../patches/sqlite-wasm-vfs", version = "0.2", optional = true }
 wasm-bindgen = "0.2"
 
 [dev-dependencies]

{ducpy-3.6.1 → ducpy-4.0.0}/packages/ducrs/build.rs

@@ -7,7 +7,12 @@ fn schema_user_version_from_sql(sql: &str) -> u32 {
     for line in sql.lines() {
         let trimmed = line.trim();
         if let Some(rest) = trimmed.strip_prefix("PRAGMA user_version") {
-            let rest = rest.trim().trim_start_matches('=').trim().trim_end_matches(';').trim();
+            let rest = rest
+                .trim()
+                .trim_start_matches('=')
+                .trim()
+                .trim_end_matches(';')
+                .trim();
             if let Ok(v) = rest.parse::<u32>() {
                 return v;
             }
@@ -133,7 +138,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             (uv, decode_user_version_to_semver(uv))
         }
         Err(e) => {
-            eprintln!("cargo:warning=Could not read {:?}: {e}. Defaulting to 0.0.0.", sql_path);
+            eprintln!(
+                "cargo:warning=Could not read {:?}: {e}. Defaulting to 0.0.0.",
+                sql_path
+            );
             (0, "0.0.0".to_string())
         }
     };
@@ -153,4 +161,4 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     println!("cargo:rerun-if-changed=build.rs");
 
     Ok(())
-}
+}

{ducpy-3.6.1 → ducpy-4.0.0}/packages/ducrs/schema/duc.sql

@@ -1,7 +1,7 @@
 -- "DUC_" in ASCII
 -- Apply in order: duc.sql → version_control.sql → search.sql
 PRAGMA application_id = 1146569567;
-PRAGMA user_version = 3000007;
+PRAGMA user_version = 4000000;
 PRAGMA journal_mode = WAL;
 PRAGMA foreign_keys = ON;
 PRAGMA synchronous = NORMAL;
@@ -996,8 +996,15 @@ CREATE TABLE external_file_revisions (
 CREATE INDEX idx_external_file_revisions_file ON external_file_revisions(file_id);
 
 -- Actual binary content for each revision, separated from metadata
--- so that metadata can be read without loading heavy blobs.
-CREATE TABLE external_file_revision_data (
-    revision_id TEXT PRIMARY KEY REFERENCES external_file_revisions(id) ON DELETE CASCADE,
-    data        BLOB NOT NULL
-) WITHOUT ROWID;
+-- and chunked so large files can be streamed without loading one BLOB.
+CREATE TABLE external_file_revision_chunks (
+    revision_id  TEXT    NOT NULL REFERENCES external_file_revisions(id) ON DELETE CASCADE,
+    chunk_index  INTEGER NOT NULL CHECK (chunk_index >= 0),
+    offset_bytes INTEGER NOT NULL CHECK (offset_bytes >= 0),
+    size_bytes   INTEGER NOT NULL CHECK (size_bytes >= 0),
+    data         BLOB    NOT NULL,
+    PRIMARY KEY (revision_id, chunk_index)
+) WITHOUT ROWID;
+
+CREATE INDEX idx_external_file_revision_chunks_revision_offset
+    ON external_file_revision_chunks(revision_id, offset_bytes);

{ducpy-3.6.1 → ducpy-4.0.0}/packages/ducrs/schema/migrations/3000001_to_3000002.sql

@@ -3,17 +3,18 @@
 -- `external_file_revision_data` table so metadata can be read without
 -- loading heavy blobs.
 
--- 1. Create the new data table
-CREATE TABLE external_file_revision_data (
-    revision_id TEXT PRIMARY KEY REFERENCES external_file_revisions(id) ON DELETE CASCADE,
+PRAGMA foreign_keys = OFF;
+
+-- 1. Stage existing data blobs before the parent table is renamed/recreated.
+CREATE TABLE _external_file_revision_data_v3000001 (
+    revision_id TEXT PRIMARY KEY,
     data        BLOB NOT NULL
 ) WITHOUT ROWID;
 
--- 2. Move existing data blobs into the new table
-INSERT INTO external_file_revision_data (revision_id, data)
+INSERT INTO _external_file_revision_data_v3000001 (revision_id, data)
     SELECT id, data FROM external_file_revisions;
 
--- 3. Recreate external_file_revisions without the data column
+-- 2. Recreate external_file_revisions without the data column.
 ALTER TABLE external_file_revisions RENAME TO _ext_revisions_old_v3000001;
 
 CREATE TABLE external_file_revisions (
@@ -35,6 +36,17 @@ INSERT INTO external_file_revisions (id, file_id, size_bytes, checksum, source_n
     SELECT id, file_id, size_bytes, checksum, source_name, mime_type, message, created, last_retrieved
     FROM _ext_revisions_old_v3000001;
 
+-- 3. Create the final data table after the new parent table exists.
+CREATE TABLE external_file_revision_data (
+    revision_id TEXT PRIMARY KEY REFERENCES external_file_revisions(id) ON DELETE CASCADE,
+    data        BLOB NOT NULL
+) WITHOUT ROWID;
+
+INSERT INTO external_file_revision_data (revision_id, data)
+    SELECT revision_id, data FROM _external_file_revision_data_v3000001;
+
 DROP TABLE _ext_revisions_old_v3000001;
+DROP TABLE _external_file_revision_data_v3000001;
 
+PRAGMA foreign_keys = ON;
 PRAGMA user_version = 3000002;

ducpy-4.0.0/packages/ducrs/schema/migrations/3000007_to_3000008.sql

@@ -0,0 +1,59 @@
+-- Migration: 3000007 -> 3000008
+-- Store external file revision bytes in ordered chunks instead of one BLOB row.
+
+BEGIN IMMEDIATE;
+PRAGMA defer_foreign_keys = ON;
+
+CREATE TABLE external_file_revision_chunks (
+    revision_id  TEXT    NOT NULL REFERENCES external_file_revisions(id) ON DELETE CASCADE,
+    chunk_index  INTEGER NOT NULL CHECK (chunk_index >= 0),
+    offset_bytes INTEGER NOT NULL CHECK (offset_bytes >= 0),
+    size_bytes   INTEGER NOT NULL CHECK (size_bytes >= 0),
+    data         BLOB    NOT NULL,
+    PRIMARY KEY (revision_id, chunk_index)
+) WITHOUT ROWID;
+
+CREATE INDEX idx_external_file_revision_chunks_revision_offset
+    ON external_file_revision_chunks(revision_id, offset_bytes);
+
+WITH RECURSIVE chunk_offsets(offset_bytes) AS (
+    VALUES(0)
+    UNION ALL
+    SELECT offset_bytes + 8388608
+    FROM chunk_offsets
+    WHERE EXISTS (
+        SELECT 1
+        FROM external_file_revision_data
+        WHERE length(data) > offset_bytes + 8388608
+    )
+)
+INSERT INTO external_file_revision_chunks (
+    revision_id, chunk_index, offset_bytes, size_bytes, data
+)
+SELECT
+    source.revision_id,
+    chunk_offsets.offset_bytes / 8388608,
+    chunk_offsets.offset_bytes,
+    length(substr(source.data, chunk_offsets.offset_bytes + 1, 8388608)),
+    substr(source.data, chunk_offsets.offset_bytes + 1, 8388608)
+FROM external_file_revision_data AS source
+JOIN chunk_offsets
+    ON chunk_offsets.offset_bytes = 0
+    OR chunk_offsets.offset_bytes < length(source.data);
+
+UPDATE external_file_revisions
+SET size_bytes = (
+    SELECT length(source.data)
+    FROM external_file_revision_data AS source
+    WHERE source.revision_id = external_file_revisions.id
+)
+WHERE EXISTS (
+    SELECT 1
+    FROM external_file_revision_data AS source
+    WHERE source.revision_id = external_file_revisions.id
+);
+
+DROP TABLE external_file_revision_data;
+
+PRAGMA user_version = 3000008;
+COMMIT;