sqlbuild-cli 0.158.0__tar.gz → 0.158.2__tar.gz

{sqlbuild_cli-0.158.0 → sqlbuild_cli-0.158.2}/Cargo.lock

@@ -878,26 +878,6 @@ dependencies = [
  "crossbeam-utils",
 ]
 
-[[package]]
-name = "recursive"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e"
-dependencies = [
- "recursive-proc-macro-impl",
- "stacker",
-]
-
-[[package]]
-name = "recursive-proc-macro-impl"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b"
-dependencies = [
- "quote",
- "syn 2.0.119",
-]
-
 [[package]]
 name = "regex"
 version = "1.13.1"
@@ -1142,7 +1122,7 @@ checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b"
 
 [[package]]
 name = "sqlbuild-analysis"
-version = "0.158.0"
+version = "0.158.2"
 dependencies = [
  "polyglot-sql",
  "polyglot-sql-function-catalogs",
@@ -1164,7 +1144,7 @@ dependencies = [
 
 [[package]]
 name = "sqlbuild-attachments"
-version = "0.158.0"
+version = "0.158.2"
 dependencies = [
  "sqlbuild-core",
  "sqlbuild-sqltext",
@@ -1172,7 +1152,7 @@ dependencies = [
 
 [[package]]
 name = "sqlbuild-cache"
-version = "0.158.0"
+version = "0.158.2"
 dependencies = [
  "blake3",
  "rayon",
@@ -1185,7 +1165,7 @@ dependencies = [
 
 [[package]]
 name = "sqlbuild-cli"
-version = "0.158.0"
+version = "0.158.2"
 dependencies = [
  "blake3",
  "ctrlc",
@@ -1200,7 +1180,7 @@ dependencies = [
 
 [[package]]
 name = "sqlbuild-config"
-version = "0.158.0"
+version = "0.158.2"
 dependencies = [
  "saphyr-parser",
  "sqlbuild-core",
@@ -1211,7 +1191,7 @@ dependencies = [
 
 [[package]]
 name = "sqlbuild-core"
-version = "0.158.0"
+version = "0.158.2"
 dependencies = [
  "regex",
  "sha2",
@@ -1219,14 +1199,14 @@ dependencies = [
 
 [[package]]
 name = "sqlbuild-cursor"
-version = "0.158.0"
+version = "0.158.2"
 dependencies = [
  "sqlbuild-core",
 ]
 
 [[package]]
 name = "sqlbuild-discovery"
-version = "0.158.0"
+version = "0.158.2"
 dependencies = [
  "junction",
  "rayon",
@@ -1239,7 +1219,7 @@ dependencies = [
 
 [[package]]
 name = "sqlbuild-model-config"
-version = "0.158.0"
+version = "0.158.2"
 dependencies = [
  "regex",
  "sqlbuild-core",
@@ -1247,7 +1227,7 @@ dependencies = [
 
 [[package]]
 name = "sqlbuild-pysource"
-version = "0.158.0"
+version = "0.158.2"
 dependencies = [
  "ruff_python_ast",
  "ruff_python_parser",
@@ -1258,7 +1238,7 @@ dependencies = [
 
 [[package]]
 name = "sqlbuild-python"
-version = "0.158.0"
+version = "0.158.2"
 dependencies = [
  "fensu-policy",
  "mimalloc",
@@ -1282,11 +1262,12 @@ dependencies = [
  "sqlbuild-scopes",
  "sqlbuild-sqltext",
  "sqlbuild-state",
+ "stacker",
 ]
 
 [[package]]
 name = "sqlbuild-refactor"
-version = "0.158.0"
+version = "0.158.2"
 dependencies = [
  "serde",
  "serde_json",
@@ -1300,7 +1281,7 @@ dependencies = [
 
 [[package]]
 name = "sqlbuild-render"
-version = "0.158.0"
+version = "0.158.2"
 dependencies = [
  "serde_json",
  "sqlbuild-cache",
@@ -1310,7 +1291,7 @@ dependencies = [
 
 [[package]]
 name = "sqlbuild-rules"
-version = "0.158.0"
+version = "0.158.2"
 dependencies = [
  "blake3",
  "fensu-policy",
@@ -1327,7 +1308,6 @@ dependencies = [
  "sqlbuild-core",
  "sqlbuild-scopes",
  "sqlbuild-sqltext",
- "sqlparser",
  "stacker",
  "tempfile",
  "toml",
@@ -1335,7 +1315,7 @@ dependencies = [
 
 [[package]]
 name = "sqlbuild-scopes"
-version = "0.158.0"
+version = "0.158.2"
 dependencies = [
  "serde_json",
  "sha2",
@@ -1348,17 +1328,16 @@ dependencies = [
 
 [[package]]
 name = "sqlbuild-sqltext"
-version = "0.158.0"
+version = "0.158.2"
 dependencies = [
  "rayon",
  "serde",
  "sqlbuild-core",
- "sqlparser",
 ]
 
 [[package]]
 name = "sqlbuild-state"
-version = "0.158.0"
+version = "0.158.2"
 dependencies = [
  "serde_json",
  "sha2",
@@ -1366,28 +1345,6 @@ dependencies = [
  "sqlbuild-cursor",
 ]
 
-[[package]]
-name = "sqlparser"
-version = "0.58.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec4b661c54b1e4b603b37873a18c59920e4c51ea8ea2cf527d925424dbd4437c"
-dependencies = [
- "log",
- "recursive",
- "sqlparser_derive",
-]
-
-[[package]]
-name = "sqlparser_derive"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.119",
-]
-
 [[package]]
 name = "stacker"
 version = "0.1.25"

{sqlbuild_cli-0.158.0 → sqlbuild_cli-0.158.2}/Cargo.toml

@@ -29,7 +29,7 @@ python-boundary = "sqlbuild-python"
 polyglot-floor = "sqlbuild-analysis"
 
 [workspace.package]
-version = "0.158.0"
+version = "0.158.2"
 edition = "2024"
 license = "Apache-2.0"
 publish = false
@@ -92,7 +92,6 @@ sqlbuild-scopes = { path = "crates/sqlbuild-scopes" }
 sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" }
 sqlbuild-state = { path = "crates/sqlbuild-state" }
 stacker = "0.1"
-sqlparser = { version = "0.58", features = ["visitor"] }
 syn = { version = "2", features = ["full", "visit", "extra-traits"] }
 tempfile = "3"
 toml = "0.9"

{sqlbuild_cli-0.158.0 → sqlbuild_cli-0.158.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlbuild-cli
-Version: 0.158.0
+Version: 0.158.2
 License-File: LICENSE
 Summary: The native sqb and sqlbuild executables of SQLBuild
 License-Expression: Apache-2.0
@@ -27,6 +27,8 @@ and safe renames to your SQL, so change stops being risky. It is a free, open-so
 SQL and Python data pipelines, and it keeps its state in append-only tables in your own warehouse:
 no external state database, no manifest files and no paid tier.
 
+Its compiler and planner run in Rust. Warehouse execution and extensions use Python.
+
 <p align="center">
   <img src="https://raw.githubusercontent.com/chio-labs/sqlbuild/main/.github/demos/rename.gif" alt="Moving and renaming an incremental model with sqb mv: sqb plan migrates the existing table instead of rebuilding it" width="100%">
 </p>

{sqlbuild_cli-0.158.0 → sqlbuild_cli-0.158.2}/README.md

@@ -18,6 +18,8 @@ and safe renames to your SQL, so change stops being risky. It is a free, open-so
 SQL and Python data pipelines, and it keeps its state in append-only tables in your own warehouse:
 no external state database, no manifest files and no paid tier.
 
+Its compiler and planner run in Rust. Warehouse execution and extensions use Python.
+
 <p align="center">
   <img src="https://raw.githubusercontent.com/chio-labs/sqlbuild/main/.github/demos/rename.gif" alt="Moving and renaming an incremental model with sqb mv: sqb plan migrates the existing table instead of rebuilding it" width="100%">
 </p>

{sqlbuild_cli-0.158.0 → sqlbuild_cli-0.158.2}/crates/sqlbuild-cache/src/artifacts/_helpers/files.rs

@@ -141,23 +141,50 @@ pub(crate) fn remove_stale(
     compiled_dir: &Path,
     managed: &HashSet<PathBuf>,
 ) -> Result<usize, ArtifactError> {
+    remove_stale_listing(compiled_dir, managed).map(|(removed, _)| removed)
+}
+
+/// `remove_stale`, with every directory left after the walk, or `None` when one was not read.
+pub(crate) fn remove_stale_listing(
+    compiled_dir: &Path,
+    managed: &HashSet<PathBuf>,
+) -> Result<(usize, Option<Vec<PathBuf>>), ArtifactError> {
     if !compiled_dir.is_dir() {
-        return Ok(0);
+        return Ok((0, None));
     }
-    let (_, removed) = remove_stale_under(compiled_dir, managed)?;
+    let cleaned: Cleaned = remove_stale_under(compiled_dir, managed)?;
     let _ = std::fs::remove_dir(compiled_dir);
-    Ok(removed)
+    let listed: Option<Vec<PathBuf>> = cleaned.complete.then_some(cleaned.directories);
+    Ok((cleaned.removed, listed))
 }
 
-/// Clean one directory bottom-up as `os.walk(topdown=False)`; returns (gone, files removed).
+/// What cleaning one directory did, and whether every directory below it could be read.
+#[derive(Debug)]
+struct Cleaned {
+    gone: bool,
+    removed: usize,
+    directories: Vec<PathBuf>,
+    complete: bool,
+}
+
+/// Clean one directory bottom-up as `os.walk(topdown=False)`.
 fn remove_stale_under(
     directory: &Path,
     managed: &HashSet<PathBuf>,
-) -> Result<(bool, usize), ArtifactError> {
+) -> Result<Cleaned, ArtifactError> {
     let entries = match std::fs::read_dir(directory) {
         Ok(entries) => entries,
-        Err(_) => return Ok((false, 0)),
+        Err(_) => {
+            return Ok(Cleaned {
+                gone: false,
+                removed: 0,
+                directories: Vec::new(),
+                complete: false,
+            });
+        }
     };
+    let mut directories: Vec<PathBuf> = vec![directory.to_path_buf()];
+    let mut complete: bool = true;
     let mut removed: usize = 0;
     let mut kept_file: bool = false;
     let mut subdirectories_removed: bool = true;
@@ -176,9 +203,11 @@ fn remove_stale_under(
             subdirectories_removed = false;
             continue;
         }
-        let (gone, below) = remove_stale_under(&subdirectory, managed)?;
-        removed += below;
-        subdirectories_removed &= gone;
+        let below: Cleaned = remove_stale_under(&subdirectory, managed)?;
+        removed += below.removed;
+        subdirectories_removed &= below.gone;
+        complete &= below.complete;
+        directories.extend(below.directories);
     }
     for file in files {
         if managed.contains(&file) {
@@ -189,7 +218,15 @@ fn remove_stale_under(
         }
     }
     let gone: bool = !kept_file && subdirectories_removed && removed_directory(directory);
-    Ok((gone, removed))
+    if gone {
+        directories.clear();
+    }
+    Ok(Cleaned {
+        gone,
+        removed,
+        directories,
+        complete,
+    })
 }
 
 fn removed_directory(directory: &Path) -> bool {

{sqlbuild_cli-0.158.0 → sqlbuild_cli-0.158.2}/crates/sqlbuild-cache/src/artifacts/_helpers/mod.rs

@@ -1,2 +1,3 @@
 pub(crate) mod files;
 pub(crate) mod sql_test_records;
+pub(crate) mod tree_stamp;

sqlbuild_cli-0.158.2/crates/sqlbuild-cache/src/artifacts/_helpers/tree_stamp.rs

@@ -0,0 +1,162 @@
+//! The stamp of a compiled tree that held exactly its managed files when last cleaned.
+
+use std::collections::HashSet;
+use std::path::{Path, PathBuf};
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use crate::digest::main::content_digest::content_digest;
+use crate::digest::types::ContentDigest;
+use crate::store::main::open_native_store::open_native_store;
+use crate::store::models::{NativeStore, RecordReader, RecordWriter};
+
+const STORE_KIND: &str = "compiled-tree";
+const STORE_ENVIRONMENT: &str = "compiled-tree-1";
+
+/// One folder's identity: its path, inode, and modification and change times.
+#[derive(Clone, Debug, PartialEq, Eq)]
+struct FolderStamp {
+    path: String,
+    inode: u64,
+    mtime_ns: i64,
+    ctime_ns: i64,
+}
+
+fn entry_key() -> ContentDigest {
+    content_digest(&["tree"])
+}
+
+/// The digest of the compiled folder and its managed file set, in any order.
+pub(crate) fn managed_digest(compiled_dir: &Path, managed: &HashSet<PathBuf>) -> ContentDigest {
+    let mut paths: Vec<String> = managed
+        .iter()
+        .map(|path| path.to_string_lossy().into_owned())
+        .collect();
+    paths.sort_unstable();
+    let mut hasher: blake3::Hasher = blake3::Hasher::new();
+    for part in std::iter::once(compiled_dir.to_string_lossy().into_owned()).chain(paths) {
+        let _ = hasher.update(&(part.len() as u64).to_le_bytes());
+        let _ = hasher.update(part.as_bytes());
+    }
+    *hasher.finalize().as_bytes()
+}
+
+#[cfg(unix)]
+fn folder_stamp(path: &Path) -> Option<FolderStamp> {
+    use std::os::unix::fs::MetadataExt;
+
+    let metadata: std::fs::Metadata = match std::fs::symlink_metadata(path) {
+        Ok(metadata) => metadata,
+        Err(_absent) => return None,
+    };
+    if !metadata.is_dir() {
+        return None;
+    }
+    Some(FolderStamp {
+        path: path.to_string_lossy().into_owned(),
+        inode: metadata.ino(),
+        mtime_ns: metadata.mtime() * 1_000_000_000 + metadata.mtime_nsec(),
+        ctime_ns: metadata.ctime() * 1_000_000_000 + metadata.ctime_nsec(),
+    })
+}
+
+#[cfg(not(unix))]
+fn folder_stamp(_path: &Path) -> Option<FolderStamp> {
+    None
+}
+
+/// Whether the tree recorded at `stamp_path` for `digest` is unchanged on disk.
+pub(crate) fn tree_unchanged(stamp_path: &Path, digest: &ContentDigest) -> bool {
+    let Ok(mut store) = open_native_store(stamp_path, STORE_KIND, STORE_ENVIRONMENT) else {
+        return false;
+    };
+    let Some(bytes) = store.get(&entry_key()) else {
+        return false;
+    };
+    let Ok((recorded, folders)) = decode(bytes) else {
+        return false;
+    };
+    recorded == *digest
+        && !folders.is_empty()
+        && folders
+            .iter()
+            .all(|folder| folder_stamp(Path::new(&folder.path)).as_ref() == Some(folder))
+}
+
+/// Remove the recorded stamp, before the tree is changed.
+pub(crate) fn forget_tree(stamp_path: &Path) {
+    let _ = std::fs::remove_file(stamp_path);
+}
+
+/// The current time in nanoseconds since the epoch, as file stamps record it.
+pub(crate) fn now_ns() -> i64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map_or(0, |elapsed| {
+            i64::try_from(elapsed.as_nanos()).unwrap_or(i64::MAX)
+        })
+}
+
+/// Record `folders` as the tree `digest` names when each stamps and changed before `trusted_ns`.
+pub(crate) fn record_tree(
+    stamp_path: &Path,
+    digest: &ContentDigest,
+    folders: &[PathBuf],
+    trusted_ns: i64,
+) {
+    let Some(stamps) = folders
+        .iter()
+        .map(|path| folder_stamp(path))
+        .collect::<Option<Vec<FolderStamp>>>()
+    else {
+        return;
+    };
+    let settled: bool = !stamps.is_empty()
+        && stamps
+            .iter()
+            .all(|stamp| stamp.mtime_ns.max(stamp.ctime_ns) < trusted_ns);
+    if !settled {
+        return;
+    }
+    let mut store: NativeStore = NativeStore {
+        kind: STORE_KIND.to_owned(),
+        environment: STORE_ENVIRONMENT.to_owned(),
+        ..NativeStore::default()
+    };
+    store.put(entry_key(), encode(digest, &stamps));
+    let _ = store.save(stamp_path, &[]);
+}
+
+fn encode(digest: &ContentDigest, folders: &[FolderStamp]) -> Vec<u8> {
+    let mut writer: RecordWriter = RecordWriter::default();
+    writer.put_bytes(digest);
+    writer.put_u64(folders.len() as u64);
+    for folder in folders {
+        writer.put_str(&folder.path);
+        writer.put_u64(folder.inode);
+        writer.put_u64(folder.mtime_ns as u64);
+        writer.put_u64(folder.ctime_ns as u64);
+    }
+    writer.into_bytes()
+}
+
+fn decode(
+    bytes: &[u8],
+) -> Result<(ContentDigest, Vec<FolderStamp>), crate::store::errors::StoreDecodeError> {
+    let mut reader: RecordReader<'_> = RecordReader::new(bytes);
+    let digest: ContentDigest = reader
+        .bytes()?
+        .try_into()
+        .map_err(|_| crate::store::errors::StoreDecodeError)?;
+    let count: u64 = reader.u64()?;
+    let mut folders: Vec<FolderStamp> = Vec::new();
+    for _ in 0..count {
+        folders.push(FolderStamp {
+            path: reader.string()?,
+            inode: reader.u64()?,
+            mtime_ns: reader.u64()? as i64,
+            ctime_ns: reader.u64()? as i64,
+        });
+    }
+    reader.finish()?;
+    Ok((digest, folders))
+}

{sqlbuild_cli-0.158.0 → sqlbuild_cli-0.158.2}/crates/sqlbuild-cache/src/artifacts/main/mod.rs

@@ -1,6 +1,7 @@
 pub mod new_sql_test_record;
 pub mod publish_staged;
 pub mod remove_stale;
+pub mod remove_stale_unless_unchanged;
 pub mod reusable_sql_test_record;
 pub mod staged_files;
 pub mod stored_sql_test_record;

sqlbuild_cli-0.158.2/crates/sqlbuild-cache/src/artifacts/main/remove_stale_unless_unchanged.rs

@@ -0,0 +1,29 @@
+use std::collections::HashSet;
+use std::path::{Path, PathBuf};
+
+use crate::artifacts::_helpers::files::remove_stale_listing;
+use crate::artifacts::_helpers::tree_stamp::{
+    forget_tree, managed_digest, now_ns, record_tree, tree_unchanged,
+};
+use crate::artifacts::errors::ArtifactError;
+use crate::digest::types::ContentDigest;
+
+/// `remove_stale`, skipped while the tree stamped at `stamp_path` still holds exactly `managed`.
+pub fn remove_stale_unless_unchanged(
+    compiled_dir: &Path,
+    managed: &HashSet<PathBuf>,
+    stamp_path: &Path,
+    racy_window_ns: i64,
+) -> Result<usize, ArtifactError> {
+    let digest: ContentDigest = managed_digest(compiled_dir, managed);
+    if tree_unchanged(stamp_path, &digest) {
+        return Ok(0);
+    }
+    forget_tree(stamp_path);
+    let trusted_ns: i64 = now_ns().saturating_sub(racy_window_ns);
+    let (removed, folders) = remove_stale_listing(compiled_dir, managed)?;
+    if let Some(folders) = folders {
+        record_tree(stamp_path, &digest, &folders, trusted_ns);
+    }
+    Ok(removed)
+}

sqlbuild_cli-0.158.2/crates/sqlbuild-cache/src/artifacts/tests/helpers.rs

@@ -0,0 +1,49 @@
+use std::path::{Path, PathBuf};
+
+/// `count` distinct SQL artifacts spread over nested directories under `root`.
+pub(crate) fn batch(root: &Path, count: usize) -> Vec<(PathBuf, Vec<u8>)> {
+    (0..count)
+        .map(|index| {
+            (
+                root.join(format!("group_{}/nested/model_{index}.sql", index % 7)),
+                format!("select {index}\n").into_bytes(),
+            )
+        })
+        .collect()
+}
+
+/// Create `path` with `contents`, including its parent directories.
+pub(crate) fn place(path: &Path, contents: &[u8]) {
+    std::fs::create_dir_all(path.parent().expect("artifact parent")).expect("artifact directory");
+    std::fs::write(path, contents).expect("existing artifact");
+}
+
+/// Every file below `root` as a sorted relative path.
+pub(crate) fn tree_files(root: &Path) -> Vec<String> {
+    let mut files: Vec<String> = walkdir::WalkDir::new(root)
+        .into_iter()
+        .flatten()
+        .filter(|entry| entry.file_type().is_file())
+        .filter_map(|entry| {
+            entry
+                .path()
+                .strip_prefix(root)
+                .ok()
+                .map(|relative| relative.to_string_lossy().into_owned())
+        })
+        .collect();
+    files.sort();
+    files
+}
+
+/// A compiled folder that is a plain directory.
+pub(crate) fn plain_compiled_root(target: &Path) {
+    std::fs::create_dir_all(target.join("compiled")).expect("compiled folder");
+}
+
+/// A compiled folder that is a link to a directory elsewhere in the target folder.
+#[cfg(unix)]
+pub(crate) fn linked_compiled_root(target: &Path) {
+    std::fs::create_dir_all(target.join("elsewhere")).expect("linked folder");
+    std::os::unix::fs::symlink(target.join("elsewhere"), target.join("compiled")).expect("link");
+}