sqlload-cli 0.3.0__tar.gz → 0.4.0__tar.gz

{sqlload_cli-0.3.0 → sqlload_cli-0.4.0}/Cargo.lock

@@ -204,6 +204,21 @@ dependencies = [
  "arrow-schema",
 ]
 
+[[package]]
+name = "arrow-csv"
+version = "60.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e4572af05fadde76a063139805c089d0915b7eb64ed5bef14456c6e458a80b28"
+dependencies = [
+ "arrow-array",
+ "arrow-cast",
+ "arrow-schema",
+ "chrono",
+ "csv",
+ "csv-core",
+ "regex",
+]
+
 [[package]]
 name = "arrow-data"
 version = "60.1.0"
@@ -745,6 +760,8 @@ dependencies = [
  "compression-core",
  "flate2",
  "memchr",
+ "zstd",
+ "zstd-safe",
 ]
 
 [[package]]
@@ -875,6 +892,27 @@ dependencies = [
  "hybrid-array",
 ]
 
+[[package]]
+name = "csv"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938"
+dependencies = [
+ "csv-core",
+ "itoa",
+ "ryu",
+ "serde_core",
+]
+
+[[package]]
+name = "csv-core"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "ctutils"
 version = "0.4.8"
@@ -3670,18 +3708,22 @@ dependencies = [
 
 [[package]]
 name = "sqlload"
-version = "0.3.0"
+version = "0.4.0"
 dependencies = [
  "aes",
  "arrow-array",
  "arrow-cast",
+ "arrow-csv",
  "arrow-schema",
+ "arrow-select",
+ "async-compression",
  "axum",
  "base64 0.22.1",
  "bytes",
  "chrono",
  "chrono-tz",
  "clap",
+ "csv-core",
  "duckdb",
  "flate2",
  "futures-util",
@@ -3714,11 +3756,12 @@ dependencies = [
  "tracing",
  "tracing-subscriber",
  "uuid",
+ "zstd",
 ]
 
 [[package]]
 name = "sqlload-testkit"
-version = "0.3.0"
+version = "0.4.0"
 dependencies = [
  "axum",
  "bytes",

{sqlload_cli-0.3.0 → sqlload_cli-0.4.0}/Cargo.toml

@@ -3,7 +3,7 @@ members = ["crates/sqlload", "crates/sqlload-testkit"]
 resolver = "2"
 
 [workspace.package]
-version = "0.3.0" # x-release-please-version
+version = "0.4.0" # x-release-please-version
 edition = "2024"
 license = "Apache-2.0"
 publish = false
@@ -14,7 +14,10 @@ rust-version = "1.95"
 aes = "0.8"
 arrow-array = "60"
 arrow-cast = "60"
+arrow-csv = "60"
 arrow-schema = "60"
+arrow-select = "60"
+async-compression = { version = "0.4", features = ["tokio", "gzip", "zstd"] }
 axum = { version = "0.8", default-features = false, features = ["http1", "tokio"] }
 base64 = "0.22"
 bytes = "1"
@@ -22,6 +25,7 @@ chrono = { version = "0.4", default-features = false, features = ["std"] }
 chrono-tz = { version = "0.10", default-features = false }
 duckdb = { version = "1.20000", features = ["bundled", "appender-arrow"] }
 clap = { version = "4", features = ["derive", "env"] }
+csv-core = "0.1"
 flate2 = "1"
 futures-util = { version = "0.3", default-features = false, features = ["std"] }
 getrandom = "0.3"
@@ -56,6 +60,7 @@ tower-service = "0.3"
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["fmt", "json", "env-filter"] }
 uuid = { version = "1", features = ["v4"] }
+zstd = "0.14"
 
 [workspace.lints.rust]
 unsafe_code = "forbid"

{sqlload_cli-0.3.0 → sqlload_cli-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlload-cli
-Version: 0.3.0
+Version: 0.4.0
 Classifier: Development Status :: 2 - Pre-Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License

{sqlload_cli-0.3.0 → sqlload_cli-0.4.0}/crates/sqlload/Cargo.toml

@@ -19,7 +19,7 @@ required-features = ["cli"]
 doc = false
 
 [features]
-default = ["cli"]
+default = ["cli", "duckdb"]
 # The `sqlload` executable and its command-line parsing.
 cli = ["dep:clap", "dep:rlimit", "dep:tracing-subscriber"]
 # TLS and SASL support in the bundled librdkafka.
@@ -29,20 +29,24 @@ ssl-vendored = ["rdkafka/ssl-vendored"]
 sasl = ["rdkafka/sasl"]
 # Select the Docker-backed integration tests (`make test-containers`).
 container-tests = []
-# DuckDB as a loader destination. Off by default: the bundled DuckDB takes many minutes to compile.
+# DuckDB as a loader destination, compiled from the bundled source (cached in CI after the first build).
 duckdb = ["dep:duckdb"]
 
 [dependencies]
 aes.workspace = true
 arrow-array = { workspace = true, features = ["chrono-tz"] }
 arrow-cast.workspace = true
+arrow-csv.workspace = true
 arrow-schema.workspace = true
+arrow-select.workspace = true
+async-compression.workspace = true
 axum.workspace = true
 base64.workspace = true
 bytes.workspace = true
 chrono.workspace = true
 chrono-tz.workspace = true
 clap = { workspace = true, optional = true }
+csv-core.workspace = true
 duckdb = { workspace = true, optional = true }
 flate2.workspace = true
 futures-util.workspace = true
@@ -65,9 +69,9 @@ serde.workspace = true
 serde_json.workspace = true
 sha2.workspace = true
 thiserror.workspace = true
-tokio.workspace = true
+tokio = { workspace = true, features = ["io-util"] }
 tokio-postgres.workspace = true
-tokio-util.workspace = true
+tokio-util = { workspace = true, features = ["io"] }
 toml.workspace = true
 tower-layer.workspace = true
 tower-service.workspace = true
@@ -83,6 +87,7 @@ serde_json.workspace = true
 sqlload-testkit.workspace = true
 tokio.workspace = true
 tokio-util.workspace = true
+zstd.workspace = true
 
 [lints]
 workspace = true

{sqlload_cli-0.3.0 → sqlload_cli-0.4.0}/crates/sqlload/src/column_types/_helpers/warehouse_names.rs

@@ -1,6 +1,7 @@
 use crate::column_types::constants::{
     BIGQUERY_NUMERIC_MAX_INTEGER_DIGITS, BIGQUERY_NUMERIC_MAX_SCALE, SQL_SERVER_MAX_NVARCHAR_LENGTH,
 };
+use crate::column_types::main::arrow_field::arrow_field;
 use crate::column_types::models::{ColumnType, Warehouse, WarehouseType};
 
 const UUID_AS_TEXT: &str = "stored as text";
@@ -208,6 +209,17 @@ pub(crate) fn sqlite(column_type: &ColumnType) -> WarehouseType {
     }
 }
 
+/// Parquet files, spelled as the Arrow type the files store; a `native` type has none.
+pub(crate) fn parquet(column_type: &ColumnType) -> WarehouseType {
+    match arrow_field("", column_type) {
+        Some(field) => exact(field.data_type().to_string()),
+        None => lossy(
+            column_type.to_string(),
+            "a native type has no Arrow type, so Parquet files cannot hold it",
+        ),
+    }
+}
+
 /// The mapping for `warehouse`.
 pub(crate) fn for_warehouse(warehouse: Warehouse) -> fn(&ColumnType) -> WarehouseType {
     match warehouse {
@@ -219,5 +231,6 @@ pub(crate) fn for_warehouse(warehouse: Warehouse) -> fn(&ColumnType) -> Warehous
         Warehouse::ClickHouse => clickhouse,
         Warehouse::SqlServer => sql_server,
         Warehouse::Sqlite => sqlite,
+        Warehouse::Parquet => parquet,
     }
 }

{sqlload_cli-0.3.0 → sqlload_cli-0.4.0}/crates/sqlload/src/column_types/models.rs

@@ -46,7 +46,7 @@ impl std::fmt::Display for ColumnType {
     }
 }
 
-/// An engine SQLLoad reads from or writes to; every engine sqlbuild has an adapter for.
+/// An engine SQLLoad reads from or writes to; every engine sqlbuild has an adapter for, plus Parquet files.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
 pub enum Warehouse {
     Snowflake,
@@ -58,6 +58,8 @@ pub enum Warehouse {
     ClickHouse,
     SqlServer,
     Sqlite,
+    /// Parquet files in an object store or directory: no SQL dialect, columns are Arrow types.
+    Parquet,
 }
 
 impl std::fmt::Display for Warehouse {
@@ -72,6 +74,7 @@ impl std::fmt::Display for Warehouse {
             Self::ClickHouse => "ClickHouse",
             Self::SqlServer => "SQL Server",
             Self::Sqlite => "SQLite",
+            Self::Parquet => "Parquet files",
         })
     }
 }

{sqlload_cli-0.3.0 → sqlload_cli-0.4.0}/crates/sqlload/src/command/_helpers/exporter_commands.rs

@@ -16,7 +16,10 @@ pub(crate) async fn run_exporter_command(
         return Err("table selection and loader flags apply to loader projects; this file is an exporter configuration".to_owned());
     }
     match command {
-        Command::Check => check(&config).await,
+        Command::Debug {
+            no_connection: true,
+        } => Ok(vec!["ok  config".to_owned()]),
+        Command::Debug { .. } => check(&config).await,
         Command::Plan(_) => plan(&config).await,
         Command::Tables { topics_file } => tables(&config, topics_file.as_deref()).await,
         Command::Apply(_) => apply(&config).await,

{sqlload_cli-0.3.0 → sqlload_cli-0.4.0}/crates/sqlload/src/command/_helpers/loader_commands.rs

@@ -1,6 +1,7 @@
 use crate::command::models::{Command, LoaderArgs};
 use crate::loader::main::apply_project::apply_project;
 use crate::loader::main::connect_targets::connect_targets;
+use crate::loader::main::debug_project::debug_project;
 use crate::loader::main::load_project::load_project;
 use crate::loader::main::plan_project::plan_project;
 use crate::loader::main::render_plan::render_plan;
@@ -61,6 +62,13 @@ fn timestamp_problem(text: &str) -> Option<String> {
 /// Run a loader command against the project in `text`.
 pub(crate) async fn run_loader(text: &str, command: Command) -> Result<Vec<String>, String> {
     let project = parse_project(text, &environment_variable).map_err(|error| error.to_string())?;
+    if let Command::Debug { no_connection } = command {
+        let (lines, ok) = debug_project(&project, !no_connection).await;
+        return match ok {
+            true => Ok(lines),
+            false => Err(lines.join("\n")),
+        };
+    }
     let destinations = connect_targets(&project)
         .await
         .map_err(|error| error.to_string())?;

{sqlload_cli-0.3.0 → sqlload_cli-0.4.0}/crates/sqlload/src/command/models.rs

@@ -26,8 +26,13 @@ pub struct Cli {
 /// The subcommands.
 #[derive(Debug, Subcommand)]
 pub enum Command {
-    /// Validate the configuration and connect to Kafka and Snowflake.
-    Check,
+    /// Validate the configuration and test every connection (`check` is an old name for it).
+    #[command(alias = "check")]
+    Debug {
+        /// Only validate the configuration; connect to nothing.
+        #[arg(long)]
+        no_connection: bool,
+    },
     /// Show what `apply` and `load` would do (exporter: each topic's table and objects).
     Plan(LoaderArgs),
     /// Print `topic<TAB>table<TAB>stream<TAB>group` per topic (`-` when skipped), without Snowflake.

sqlload_cli-0.4.0/crates/sqlload/src/file_store/main/delete_files.rs

@@ -0,0 +1,25 @@
+//! Delete files.
+
+use object_store::ObjectStoreExt;
+
+use crate::file_store::_helpers::keys::object_path;
+use crate::file_store::errors::StoreError;
+use crate::file_store::models::OpenFolder;
+
+/// Delete each of `relative_keys` under `folder`; a file already gone is not an error.
+pub async fn delete_files(folder: &OpenFolder, relative_keys: &[String]) -> Result<(), StoreError> {
+    for relative_key in relative_keys {
+        let failed = |source: object_store::Error| StoreError::Object {
+            action: "cannot delete",
+            key: relative_key.clone(),
+            source,
+        };
+        let path =
+            object_path(&format!("{}{relative_key}", folder.location.prefix)).map_err(failed)?;
+        match folder.store.delete(&path).await {
+            Ok(()) | Err(object_store::Error::NotFound { .. }) => {}
+            Err(error) => return Err(failed(error)),
+        }
+    }
+    Ok(())
+}

{sqlload_cli-0.3.0 → sqlload_cli-0.4.0}/crates/sqlload/src/file_store/main/mod.rs

@@ -1,4 +1,6 @@
+pub mod delete_files;
 pub mod list_files;
 pub mod open_folder;
+pub mod probe_folder;
 pub mod read_file;
 pub mod write_file;

sqlload_cli-0.4.0/crates/sqlload/src/file_store/main/probe_folder.rs

@@ -0,0 +1,24 @@
+//! Check that a folder can be listed.
+
+use crate::file_store::_helpers::keys::object_path;
+use crate::file_store::errors::StoreError;
+use crate::file_store::models::OpenFolder;
+
+/// The number of files and folders directly under `folder`, from one listing request.
+pub async fn probe_folder(folder: &OpenFolder) -> Result<usize, StoreError> {
+    let failed = |source: object_store::Error| StoreError::Object {
+        action: "cannot list",
+        key: folder.location.to_string(),
+        source,
+    };
+    let prefix = match folder.location.prefix.is_empty() {
+        true => None,
+        false => Some(object_path(&folder.location.prefix).map_err(failed)?),
+    };
+    let listed = folder
+        .store
+        .list_with_delimiter(prefix.as_ref())
+        .await
+        .map_err(failed)?;
+    Ok(listed.objects.len() + listed.common_prefixes.len())
+}

sqlload_cli-0.4.0/crates/sqlload/src/files_destination/_helpers/bounds.rs

@@ -0,0 +1,98 @@
+use std::time::Duration;
+
+use chrono::{Days, NaiveDateTime};
+
+use crate::column_types::models::ColumnType;
+use crate::files_destination::constants::{MIDNIGHT, UTC_SUFFIX};
+use crate::files_destination::models::PartitionBy;
+use crate::write_sql::constants::CURSOR_TIMESTAMP_FORMAT;
+use crate::write_sql::main::cursor_bound::cursor_bound;
+use crate::write_sql::models::CursorValue;
+
+fn zone(cursor_type: &ColumnType) -> &'static str {
+    match cursor_type {
+        ColumnType::TimestampTz => UTC_SUFFIX,
+        _ => "",
+    }
+}
+
+fn is_timestamp(cursor_type: &ColumnType) -> bool {
+    matches!(
+        cursor_type,
+        ColumnType::TimestampTz | ColumnType::TimestampNtz
+    )
+}
+
+fn timestamp(bound: &str) -> Result<NaiveDateTime, String> {
+    NaiveDateTime::parse_from_str(bound.trim_end_matches(UTC_SUFFIX), "%Y-%m-%d %H:%M:%S%.f")
+        .map_err(|_| format!("cursor bound {bound:?} is not a timestamp"))
+}
+
+/// The cursor value partition `value` starts at: its first instant for a timestamp cursor by date.
+pub(crate) fn partition_start(value: &str, by: PartitionBy, cursor_type: &ColumnType) -> String {
+    match (by, is_timestamp(cursor_type)) {
+        (PartitionBy::Date, true) => format!("{value} {MIDNIGHT}{}", zone(cursor_type)),
+        _ => value.to_owned(),
+    }
+}
+
+/// `bound` moved back by `lookback`; only timestamp and date cursors take a lookback.
+pub(crate) fn minus_lookback(
+    bound: String,
+    cursor_type: &ColumnType,
+    lookback: Option<Duration>,
+) -> Result<String, String> {
+    let Some(lookback) = lookback else {
+        return Ok(bound);
+    };
+    match cursor_type {
+        ColumnType::Date => cursor_bound(&bound, CursorValue::Date, Some(lookback)),
+        ColumnType::TimestampTz | ColumnType::TimestampNtz => {
+            let micros = timestamp(&bound)?.and_utc().timestamp_micros().to_string();
+            let zoned = *cursor_type == ColumnType::TimestampTz;
+            cursor_bound(&micros, CursorValue::EpochMicros { zoned }, Some(lookback))
+        }
+        other => Err(format!(
+            "lookback needs a timestamp or date cursor, but the cursor is {other}"
+        )),
+    }
+}
+
+/// `bound` moved down to the start of its partition, so whole partitions are read.
+pub(crate) fn align_down(
+    bound: &str,
+    by: PartitionBy,
+    cursor_type: &ColumnType,
+) -> Result<String, String> {
+    match (by, is_timestamp(cursor_type)) {
+        (PartitionBy::Date, true) => {
+            let date = timestamp(bound)?.date().format("%Y-%m-%d").to_string();
+            Ok(partition_start(&date, by, cursor_type))
+        }
+        _ => Ok(bound.to_owned()),
+    }
+}
+
+/// `bound` (exclusive) moved up to the start of the next partition unless it already starts one.
+pub(crate) fn align_up(
+    bound: &str,
+    by: PartitionBy,
+    cursor_type: &ColumnType,
+) -> Result<String, String> {
+    let (PartitionBy::Date, true) = (by, is_timestamp(cursor_type)) else {
+        return Ok(bound.to_owned());
+    };
+    let time = timestamp(bound)?;
+    let start = time.date().and_time(chrono::NaiveTime::MIN);
+    let aligned = match time == start {
+        true => start,
+        false => start
+            .checked_add_days(Days::new(1))
+            .ok_or_else(|| format!("cursor bound {bound:?} is out of range"))?,
+    };
+    Ok(format!(
+        "{}{}",
+        aligned.format(CURSOR_TIMESTAMP_FORMAT),
+        zone(cursor_type)
+    ))
+}

sqlload_cli-0.4.0/crates/sqlload/src/files_destination/_helpers/loads.rs

@@ -0,0 +1,8 @@
+use crate::files_destination::constants::PART_PREFIX;
+
+/// The load id in a written file's key (`.../part-<load id>-<n>.parquet`), if the file name has that shape.
+pub(crate) fn load_id_of(key: &str) -> Option<&str> {
+    let name = key.rsplit_once('/').map_or(key, |(_, name)| name);
+    let (load_id, _) = name.strip_prefix(PART_PREFIX)?.rsplit_once('-')?;
+    (!load_id.is_empty()).then_some(load_id)
+}

sqlload_cli-0.4.0/crates/sqlload/src/files_destination/_helpers/markers.rs

@@ -0,0 +1,45 @@
+use crate::file_store::models::OpenFolder;
+use crate::files_destination::constants::{PENDING_FOLDER, PENDING_REPLACING, PENDING_WRITING};
+use crate::files_destination::models::PendingPhase;
+use crate::store_paths::models::Location;
+
+/// The folder of a table's pending markers, in the same store.
+pub(crate) fn pending_folder(folder: &OpenFolder) -> OpenFolder {
+    OpenFolder {
+        store: folder.store.clone(),
+        location: Location {
+            store: folder.location.store.clone(),
+            prefix: format!("{}{PENDING_FOLDER}", folder.location.prefix),
+        },
+    }
+}
+
+/// A marker's text: the phase, then one key per line.
+pub(crate) fn marker_text(phase: PendingPhase, keys: &[String]) -> String {
+    let first = match phase {
+        PendingPhase::Writing => PENDING_WRITING,
+        PendingPhase::Replacing => PENDING_REPLACING,
+    };
+    let mut lines = vec![first.to_owned()];
+    lines.extend(keys.iter().cloned());
+    lines.join("\n")
+}
+
+/// The phase and keys a marker's text records.
+pub(crate) fn parse_marker(text: &str) -> Result<(PendingPhase, Vec<String>), String> {
+    let mut lines = text.lines();
+    let phase = match lines.next() {
+        Some(PENDING_WRITING) => PendingPhase::Writing,
+        Some(PENDING_REPLACING) => PendingPhase::Replacing,
+        other => {
+            return Err(format!(
+                "a pending marker starts with {other:?}, not {PENDING_WRITING:?} or {PENDING_REPLACING:?}"
+            ));
+        }
+    };
+    let keys: Vec<String> = lines
+        .filter(|line| !line.is_empty())
+        .map(str::to_owned)
+        .collect();
+    Ok((phase, keys))
+}

sqlload_cli-0.4.0/crates/sqlload/src/files_destination/_helpers/mod.rs

@@ -0,0 +1,10 @@
+pub(crate) mod bounds;
+pub(crate) mod loads;
+pub(crate) mod markers;
+pub(crate) mod parquet_files;
+pub(crate) mod partitions;
+pub(crate) mod shape;
+pub(crate) mod spill;
+pub(crate) mod split;
+pub(crate) mod statistics;
+pub(crate) mod template;

sqlload_cli-0.4.0/crates/sqlload/src/files_destination/_helpers/parquet_files.rs

@@ -0,0 +1,60 @@
+use arrow_schema::SchemaRef;
+use bytes::Bytes;
+use parquet::arrow::ArrowWriter;
+use parquet::basic::{Compression, ZstdLevel};
+use parquet::file::properties::WriterProperties;
+
+use crate::file_store::main::write_file::write_file;
+use crate::file_store::models::OpenFolder;
+use crate::files_destination::constants::{OPEN_FILE_OVERHEAD_BYTES, PARQUET_SUFFIX, PART_PREFIX};
+use crate::files_destination::errors::FilesDestinationError;
+use crate::files_destination::models::{FilesWriter, OpenFile};
+
+/// An empty zstd Parquet file in memory with the table's schema; statistics are never truncated, so maximums are exact.
+pub(crate) fn new_file(schema: &SchemaRef) -> Result<OpenFile, String> {
+    let properties = WriterProperties::builder()
+        .set_compression(Compression::ZSTD(ZstdLevel::default()))
+        .set_statistics_truncate_length(None)
+        .set_column_index_truncate_length(None)
+        .build();
+    ArrowWriter::try_new(Vec::new(), schema.clone(), Some(properties))
+        .map(|writer| OpenFile { writer, rows: 0 })
+        .map_err(|error| error.to_string())
+}
+
+/// The bytes an open file holds in memory: written, still buffered, and the writer's own overhead.
+pub(crate) fn open_bytes(file: &OpenFile) -> usize {
+    file.writer.bytes_written()
+        + file
+            .writer
+            .in_progress_size()
+            .max(file.writer.memory_size())
+        + OPEN_FILE_OVERHEAD_BYTES
+}
+
+/// The key of the writer's next file in partition `value`: `<partition folder>/part-<load id>-<n>.parquet`.
+pub(crate) fn next_key(writer: &FilesWriter, value: &str) -> String {
+    let folder = match &writer.partition {
+        Some(partition) => format!("{}/", partition.name(value)),
+        None => String::new(),
+    };
+    format!(
+        "{folder}{PART_PREFIX}{}-{}{PARQUET_SUFFIX}",
+        writer.load_id,
+        writer.written.len()
+    )
+}
+
+/// Finish `file` and put it at `key` in one write, so readers see it only once complete.
+pub(crate) async fn store_file(
+    folder: &OpenFolder,
+    key: &str,
+    file: OpenFile,
+) -> Result<(), FilesDestinationError> {
+    let failed = |error: parquet::errors::ParquetError| FilesDestinationError::Write {
+        key: key.to_owned(),
+        reason: error.to_string(),
+    };
+    let bytes = file.writer.into_inner().map_err(failed)?;
+    Ok(write_file(folder, key, Bytes::from(bytes)).await?)
+}