dirsql 0.3.69__tar.gz → 0.3.70__tar.gz

{dirsql-0.3.69 → dirsql-0.3.70}/Cargo.lock

@@ -500,7 +500,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql-py-ext"
-version = "0.3.69"
+version = "0.3.70"
 dependencies = [
  "dirsql",
  "pyo3",

{dirsql-0.3.69 → dirsql-0.3.70}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.3.69
+Version: 0.3.70
 Summary: Ephemeral SQL index over a local directory
 Keywords: sql,filesystem,directory,sqlite,index
 Author: Kevin Scott

{dirsql-0.3.69 → dirsql-0.3.70}/packages/python/Cargo.toml

@@ -4,7 +4,7 @@ name = "dirsql-py-ext"
 # pypi/maturin handler can rewrite it via `write-version` before
 # `maturin build`. `pyproject.toml` declares `dynamic = ["version"]`
 # and maturin reads this field. Mirrors `packages/rust/Cargo.toml`.
-version = "0.3.69"
+version = "0.3.70"
 edition.workspace = true
 publish = false
 readme = "README.md"

{dirsql-0.3.69 → dirsql-0.3.70}/packages/python/e2e-attestation.json

@@ -1,6 +1,6 @@
 {
   "command": "uv run python -m pytest tests/e2e/ -x -q",
-  "ran_at": 1783257990,
+  "ran_at": 1783387386,
   "exit_code": 0,
-  "commit": "cb8d663ecb72a808f4ab99589af16949e9449f3b"
+  "commit": "a9cf031533c564bf43c77d893745f9e85fed059f"
 }

dirsql-0.3.70/packages/rust/src/cli/execute.rs

@@ -0,0 +1,314 @@
+//! Transport-agnostic `/query` pipeline.
+//!
+//! The full orchestration for running one query — intake validation,
+//! the `pre-query` hook, the query timeout, [`DirSQL::query`], row
+//! serialization, the `post-query` hook, and error classification —
+//! lives here exactly once. The HTTP handler and the one-shot
+//! `dirsql query` subcommand (#399) are thin transport adapters over
+//! [`execute_query`], so the two surfaces cannot drift behaviorally:
+//! per-surface code only maps [`QueryFailure`] to a status code or an
+//! exit code.
+
+use std::time::Duration;
+
+use serde::Deserialize;
+use serde_json::Value;
+
+use super::serialize::rows_to_json;
+use super::{AppState, PostQuery, PreQuery};
+use crate::command::{Placeholder, run_command};
+use crate::{DirSQL, DirSqlError};
+
+/// Cap on the serialized result payload passed as the `{args}` argv token.
+/// Beyond this, `{args}` is emptied and the operator is directed to stdin
+/// (which always carries the full payload) — comfortably under Linux's 128 KiB
+/// single-arg `MAX_ARG_STRLEN`.
+const POST_QUERY_ARGS_MAX: usize = 96 * 1024;
+
+/// Why a query failed, classified independently of transport. The HTTP
+/// adapter maps each arm to a status code (400 / 408 / 500 / 503); the CLI
+/// adapter maps every arm to stderr + a non-zero exit.
+#[derive(Debug)]
+pub enum QueryFailure {
+    /// Malformed input the caller can fix: unparsable body, missing/empty
+    /// `sql`, or a SQL error from the core (HTTP 400).
+    BadRequest(String),
+    /// The query exceeded the configured timeout (HTTP 408).
+    Timeout(Duration),
+    /// A server-side fault: hook failure, join error, lock poisoning
+    /// (HTTP 500).
+    Internal(String),
+    /// The index never became ready — the degraded config state (HTTP 503).
+    Unavailable(String),
+}
+
+impl QueryFailure {
+    /// The diagnostic message, identical across transports (the HTTP
+    /// `{"error": …}` body and the CLI stderr line).
+    pub fn message(&self) -> String {
+        match self {
+            Self::Timeout(timeout) => format!("query exceeded {timeout:?} timeout"),
+            Self::BadRequest(msg) | Self::Internal(msg) | Self::Unavailable(msg) => msg.clone(),
+        }
+    }
+}
+
+#[derive(Debug, Deserialize)]
+struct QueryBody {
+    sql: Option<String>,
+}
+
+/// Run one query end to end: resolve the SQL from `raw_body` (through the
+/// `pre-query` hook when present, else parsed as `{"sql": …}`), execute it
+/// against the index under `timeout`, serialize the rows, and reshape them
+/// through the `post-query` hook when present. `raw_body` is the exact
+/// payload a `POST /query` request carries; the CLI adapter synthesizes the
+/// same shape so both surfaces share intake validation and hook semantics.
+pub async fn execute_query(
+    state: &AppState,
+    raw_body: String,
+    timeout: Duration,
+    pre_query: Option<&PreQuery>,
+    post_query: Option<&PostQuery>,
+) -> Result<Value, QueryFailure> {
+    // Resolve the SQL to run. With a `pre-query` hook the raw body is
+    // rewritten by the command; without one it is parsed as `{"sql": …}`.
+    let sql = match pre_query {
+        Some(pq) => run_pre_query(pq, raw_body).await?,
+        None => parse_sql_body(&raw_body)?,
+    };
+
+    let db = require_ready(state)?;
+
+    let join =
+        tokio::time::timeout(timeout, tokio::task::spawn_blocking(move || db.query(&sql))).await;
+
+    match join {
+        Ok(Ok(Ok(rows))) => {
+            let rows_json = rows_to_json(&rows);
+            match post_query {
+                Some(pq) => run_post_query(pq, rows_json).await,
+                None => Ok(Value::Array(rows_json)),
+            }
+        }
+        Ok(Ok(Err(err))) => Err(classify_query_error(err)),
+        Ok(Err(join_err)) => Err(QueryFailure::Internal(join_err.to_string())),
+        Err(_elapsed) => Err(QueryFailure::Timeout(timeout)),
+    }
+}
+
+/// Return a cloned [`DirSQL`] handle, or [`QueryFailure::Unavailable`] if
+/// the index started in the degraded [`AppState::Unavailable`] state.
+pub fn require_ready(state: &AppState) -> Result<DirSQL, QueryFailure> {
+    match state {
+        AppState::Ready(db) => Ok(db.clone()),
+        AppState::Unavailable(reason) => Err(QueryFailure::Unavailable(reason.clone())),
+    }
+}
+
+/// Parse a raw query body as `{"sql": …}` and return the trimmed SQL.
+/// `BadRequest` on malformed JSON, and on a missing/empty `sql` field.
+fn parse_sql_body(body: &str) -> Result<String, QueryFailure> {
+    let parsed: QueryBody =
+        serde_json::from_str(body).map_err(|err| QueryFailure::BadRequest(err.to_string()))?;
+    match parsed.sql.as_deref().map(str::trim) {
+        Some(s) if !s.is_empty() => Ok(s.to_string()),
+        Some(_) => Err(QueryFailure::BadRequest("`sql` must not be empty".into())),
+        None => Err(QueryFailure::BadRequest("missing `sql` field".into())),
+    }
+}
+
+/// SQL errors from the core are the caller's to fix (`BadRequest`); every
+/// other failure (lock poisoning, watch/config faults) is server-side
+/// (`Internal`).
+fn classify_query_error(err: DirSqlError) -> QueryFailure {
+    match err {
+        DirSqlError::Core(_) => QueryFailure::BadRequest(err.to_string()),
+        _ => QueryFailure::Internal(err.to_string()),
+    }
+}
+
+/// Run the `pre-query` hook over the raw request body and return the SQL it
+/// prints. The body is passed as the injection-safe `{args}` placeholder (a
+/// single argv token); the command's last non-empty stdout line is the SQL to
+/// run. Any failure (non-zero exit, timeout, spawn error) maps to `Internal`
+/// carrying the command's stderr tail.
+async fn run_pre_query(pq: &PreQuery, raw_body: String) -> Result<String, QueryFailure> {
+    let command = pq.command.clone();
+    let config_dir = pq.config_dir.clone();
+    let timeout = pq.timeout;
+    // `run_command` is blocking — it spawns a child and joins drain threads —
+    // so run it off the async runtime. It enforces the hook's timeout
+    // (the global `[dirsql].hook-timeout`, default 30s) internally, so no outer
+    // `tokio::time::timeout` is needed.
+    let outcome = tokio::task::spawn_blocking(move || {
+        run_command(
+            &command,
+            &[Placeholder::new("args", &raw_body)],
+            &config_dir,
+            timeout,
+            None,
+        )
+    })
+    .await
+    .map_err(|join_err| QueryFailure::Internal(join_err.to_string()))?;
+
+    // `run_command` only returns `Ok` with a non-empty last stdout line
+    // (`EmptyOutput` otherwise), so the payload is the SQL as-is.
+    outcome
+        .map(|out| out.payload)
+        .map_err(|err| QueryFailure::Internal(err.to_string()))
+}
+
+/// Run the `post-query` hook over a successful result set and return the JSON
+/// body it prints. The rows are serialized to a JSON array and delivered two
+/// ways: always on the child's stdin (unbounded, injection-safe), and as the
+/// `{args}` placeholder when the payload is within [`POST_QUERY_ARGS_MAX`]
+/// (beyond that `{args}` is emptied and a warning names the size, directing
+/// the operator to stdin — never silent truncation). The command's last
+/// non-empty stdout line is parsed as JSON and returned as the result;
+/// anything that isn't valid JSON, or any failure (non-zero exit, timeout,
+/// spawn error), maps to `Internal`.
+async fn run_post_query(pq: &PostQuery, rows: Vec<Value>) -> Result<Value, QueryFailure> {
+    let payload =
+        serde_json::to_string(&rows).map_err(|err| QueryFailure::Internal(err.to_string()))?;
+    let command = pq.command.clone();
+    let config_dir = pq.config_dir.clone();
+    let timeout = pq.timeout;
+    // `run_command` is blocking — it spawns a child and joins drain threads —
+    // so run it off the async runtime. It enforces the hook's timeout
+    // (the global `[dirsql].hook-timeout`, default 30s) internally, so no outer
+    // `tokio::time::timeout` is needed.
+    let outcome = tokio::task::spawn_blocking(move || {
+        let args_value = if payload.len() <= POST_QUERY_ARGS_MAX {
+            payload.clone()
+        } else {
+            eprintln!(
+                "dirsql: post-query result payload is {} bytes, exceeding the \
+                 {POST_QUERY_ARGS_MAX}-byte argv threshold; `{{args}}` is emptied — \
+                 read the rows from stdin instead",
+                payload.len()
+            );
+            String::new()
+        };
+        run_command(
+            &command,
+            &[Placeholder::new("args", &args_value)],
+            &config_dir,
+            timeout,
+            Some(payload.as_bytes()),
+        )
+    })
+    .await
+    .map_err(|join_err| QueryFailure::Internal(join_err.to_string()))?;
+
+    let out = outcome.map_err(|err| QueryFailure::Internal(err.to_string()))?;
+
+    // The command's payload (last non-empty stdout line) is the JSON result
+    // body; reject anything that doesn't parse as JSON.
+    serde_json::from_str(&out.payload).map_err(|err| {
+        QueryFailure::Internal(format!("post-query did not return valid JSON: {err}"))
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // The `DirSqlError::Core => BadRequest` arm of `classify_query_error` is
+    // exercised end-to-end at the integration tier by
+    // `post_query_malformed_sql_returns_400_not_500` in
+    // `tests/cli_integration.rs`, which posts malformed SQL to `/query` and
+    // asserts the 400. Constructing a `Core` value inline would require
+    // importing the first-party `crate::db::DbError`, which the
+    // `testing-conventions` `unit lint` isolation rule forbids (a unit test
+    // may reach only `super::` and pure `std`). The non-Core arm below is
+    // pure -- it builds a `super::DirSqlError::Lock` -- so it stays inline.
+
+    #[test]
+    fn classify_non_core_error_is_internal() {
+        // Lock/watch/config failures are server-side faults -> `Internal`.
+        // This drives the `_ =>` arm of `classify_query_error`.
+        let err = DirSqlError::Lock("poisoned".into());
+        let failure = classify_query_error(err);
+        assert!(
+            matches!(failure, QueryFailure::Internal(_)),
+            "got: {failure:?}"
+        );
+    }
+
+    #[test]
+    fn require_ready_fails_unavailable_when_degraded() {
+        // The degraded state yields `Unavailable` carrying the diagnostic
+        // verbatim instead of a `DirSQL` handle.
+        let state = AppState::Unavailable("config failed to load".into());
+        // `DirSQL` isn't `Debug`, so go through `.err()` (which drops the Ok
+        // value) rather than `expect_err`.
+        let failure = require_ready(&state)
+            .err()
+            .expect("Unavailable must not yield a db");
+        match failure {
+            QueryFailure::Unavailable(reason) => assert_eq!(reason, "config failed to load"),
+            other => panic!("expected Unavailable, got: {other:?}"),
+        }
+    }
+
+    // `parse_sql_body` is the no-`pre-query` intake path: it is pure (serde
+    // only), so it is unit-tested here directly rather than through the async
+    // `execute_query` pipeline (which needs a live index and is covered at
+    // the integration tier).
+
+    #[test]
+    fn parse_sql_body_returns_trimmed_sql() {
+        // Surrounding whitespace is stripped; the inner SQL is returned as-is.
+        let sql = parse_sql_body(r#"{"sql": "  SELECT 1  "}"#).expect("valid body");
+        assert_eq!(sql, "SELECT 1");
+    }
+
+    #[test]
+    fn parse_sql_body_rejects_malformed_json() {
+        // A body that isn't JSON fails at the serde step -> `BadRequest`.
+        let failure = parse_sql_body("not json").expect_err("malformed JSON must be rejected");
+        assert!(
+            matches!(failure, QueryFailure::BadRequest(_)),
+            "got: {failure:?}"
+        );
+    }
+
+    #[test]
+    fn parse_sql_body_rejects_whitespace_only_sql() {
+        // A present-but-blank `sql` trims to empty -> `BadRequest` (the
+        // `Some(_)` arm, and the `false` side of the `!s.is_empty()` guard).
+        let failure = parse_sql_body(r#"{"sql": "   "}"#).expect_err("empty sql must be rejected");
+        match failure {
+            QueryFailure::BadRequest(msg) => assert_eq!(msg, "`sql` must not be empty"),
+            other => panic!("expected BadRequest, got: {other:?}"),
+        }
+    }
+
+    #[test]
+    fn parse_sql_body_rejects_missing_sql_field() {
+        // Valid JSON object with no `sql` key -> the `None` arm -> `BadRequest`.
+        let failure = parse_sql_body("{}").expect_err("missing sql must be rejected");
+        match failure {
+            QueryFailure::BadRequest(msg) => assert_eq!(msg, "missing `sql` field"),
+            other => panic!("expected BadRequest, got: {other:?}"),
+        }
+    }
+
+    #[test]
+    fn timeout_failure_message_names_the_duration() {
+        // The timeout arm formats its message from the stored duration —
+        // the exact text the HTTP 408 body carried before the extraction.
+        let failure = QueryFailure::Timeout(Duration::from_secs(30));
+        assert_eq!(failure.message(), "query exceeded 30s timeout");
+    }
+
+    #[test]
+    fn non_timeout_failure_messages_pass_through_verbatim() {
+        // The other three arms carry their diagnostic string as-is.
+        assert_eq!(QueryFailure::BadRequest("a".into()).message(), "a");
+        assert_eq!(QueryFailure::Internal("b".into()).message(), "b");
+        assert_eq!(QueryFailure::Unavailable("c".into()).message(), "c");
+    }
+}

{dirsql-0.3.69 → dirsql-0.3.70}/packages/rust/src/cli/mod.rs

@@ -15,7 +15,9 @@
 //! submodule:
 //!
 //! - [`server`] — bind/serve/shutdown plumbing.
-//! - [`router`] — axum routes + request handlers.
+//! - [`router`] — axum routes + request handlers (thin HTTP adapters).
+//! - [`execute`] — the transport-agnostic query pipeline shared by the
+//!   HTTP handler and the one-shot `dirsql query` subcommand.
 //! - [`serialize`] — row + event → JSON.
 
 use std::net::SocketAddr;
@@ -29,6 +31,7 @@ use tokio::task::JoinHandle;
 use crate::DirSQL;
 use crate::command::DEFAULT_COMMAND_TIMEOUT;
 
+pub mod execute;
 pub mod init;
 pub mod router;
 pub mod serialize;

dirsql-0.3.70/packages/rust/src/cli/router.rs

@@ -0,0 +1,191 @@
+//! Axum router, request handlers, and shared context.
+//!
+//! Handlers are thin transport adapters: the whole query pipeline lives in
+//! [`super::execute`], and this module only maps its outcome onto HTTP —
+//! [`QueryFailure`] arms onto status codes, the success value onto a JSON
+//! body.
+
+use std::sync::Arc;
+use std::time::Duration;
+
+use axum::Json;
+use axum::Router;
+use axum::extract::State;
+use axum::http::{HeaderValue, StatusCode, header};
+use axum::response::sse::{Event as SseEvent, KeepAlive, Sse};
+use axum::response::{IntoResponse, Response};
+use axum::routing::{get, post};
+use futures::stream::StreamExt;
+use serde_json::json;
+use tokio::sync::{broadcast, watch};
+use tokio_stream::wrappers::BroadcastStream;
+
+use super::execute::{QueryFailure, execute_query, require_ready};
+use super::{AppState, PostQuery, PreQuery};
+
+pub(super) struct AppContext {
+    pub state: AppState,
+    pub events: broadcast::Sender<String>,
+    pub cancel: watch::Receiver<bool>,
+    pub query_timeout: Duration,
+    /// Optional server-wide `pre-query` hook. When `Some`, `POST /query`
+    /// rewrites the request body through the command; when `None`, the body
+    /// is parsed as `{"sql": …}`.
+    pub pre_query: Option<PreQuery>,
+    /// Optional server-wide `post-query` hook. When `Some`, a successful
+    /// `POST /query` result set is reshaped by the command before responding;
+    /// when `None`, the rows are returned as-is.
+    pub post_query: Option<PostQuery>,
+}
+
+pub(super) type SharedCtx = Arc<AppContext>;
+
+pub(super) fn router(ctx: SharedCtx) -> Router {
+    Router::new()
+        .route(
+            "/query",
+            post(handle_query).on(axum::routing::MethodFilter::GET, method_not_allowed),
+        )
+        .route(
+            "/events",
+            get(handle_events).on(axum::routing::MethodFilter::POST, method_not_allowed),
+        )
+        .with_state(ctx)
+}
+
+async fn handle_query(State(ctx): State<SharedCtx>, body: String) -> Response {
+    match execute_query(
+        &ctx.state,
+        body,
+        ctx.query_timeout,
+        ctx.pre_query.as_ref(),
+        ctx.post_query.as_ref(),
+    )
+    .await
+    {
+        Ok(value) => Json(value).into_response(),
+        Err(failure) => failure_response(&failure),
+    }
+}
+
+/// The HTTP status for each [`QueryFailure`] arm. This mapping — together
+/// with [`failure_response`]'s `{"error": …}` body — is the entirety of the
+/// HTTP adapter's own behavior; everything else is the shared pipeline.
+fn failure_status(failure: &QueryFailure) -> StatusCode {
+    match failure {
+        QueryFailure::BadRequest(_) => StatusCode::BAD_REQUEST,
+        QueryFailure::Timeout(_) => StatusCode::REQUEST_TIMEOUT,
+        QueryFailure::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR,
+        QueryFailure::Unavailable(_) => StatusCode::SERVICE_UNAVAILABLE,
+    }
+}
+
+fn failure_response(failure: &QueryFailure) -> Response {
+    error_response(failure_status(failure), failure.message())
+}
+
+async fn handle_events(State(ctx): State<SharedCtx>) -> Response {
+    if let Err(failure) = require_ready(&ctx.state) {
+        return failure_response(&failure);
+    }
+
+    // Subscribe BEFORE anything that might block so we don't drop events
+    // that fire between subscribing and the first poll.
+    let rx = ctx.events.subscribe();
+    let events = BroadcastStream::new(rx).filter_map(|res| async move {
+        match res {
+            Ok(data) => Some(Ok::<SseEvent, std::convert::Infallible>(
+                SseEvent::default().event("row").data(data),
+            )),
+            // Lagging subscriber: skip missed events rather than terminating.
+            Err(_) => None,
+        }
+    });
+
+    // Yield a ready event up front so clients have a reliable signal that
+    // the subscription is attached. Data is non-empty because SSE parsers
+    // skip events with no `data:` line.
+    let ready = futures::stream::once(async {
+        Ok::<SseEvent, std::convert::Infallible>(SseEvent::default().event("ready").data("{}"))
+    });
+    let combined = ready.chain(events);
+
+    // Close the stream when the server's cancellation signal fires so
+    // graceful shutdown actually completes (otherwise SSE streams hold
+    // axum's in-flight count at > 0 indefinitely).
+    let mut cancel = ctx.cancel.clone();
+    let stream = combined.take_until(async move {
+        let _ = cancel.wait_for(|v| *v).await;
+    });
+
+    Sse::new(stream)
+        .keep_alive(KeepAlive::default())
+        .into_response()
+}
+
+async fn method_not_allowed() -> Response {
+    (StatusCode::METHOD_NOT_ALLOWED, "method not allowed").into_response()
+}
+
+pub(super) fn error_response(status: StatusCode, message: impl Into<String>) -> Response {
+    let body = json!({ "error": message.into() });
+    let mut resp = (status, Json(body)).into_response();
+    resp.headers_mut().insert(
+        header::CONTENT_TYPE,
+        HeaderValue::from_static("application/json"),
+    );
+    resp
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // The pipeline itself (intake validation, hooks, timeout, execution,
+    // error classification) is unit-tested in `cli/execute.rs` and covered
+    // end-to-end by `tests/cli_integration.rs`. What remains here is the
+    // HTTP adapter's own behavior: the failure -> status mapping and the
+    // JSON error body.
+
+    #[test]
+    fn failure_status_maps_each_arm_to_its_http_status() {
+        assert_eq!(
+            failure_status(&QueryFailure::BadRequest("x".into())),
+            StatusCode::BAD_REQUEST
+        );
+        assert_eq!(
+            failure_status(&QueryFailure::Timeout(Duration::from_secs(30))),
+            StatusCode::REQUEST_TIMEOUT
+        );
+        assert_eq!(
+            failure_status(&QueryFailure::Internal("x".into())),
+            StatusCode::INTERNAL_SERVER_ERROR
+        );
+        assert_eq!(
+            failure_status(&QueryFailure::Unavailable("x".into())),
+            StatusCode::SERVICE_UNAVAILABLE
+        );
+    }
+
+    #[test]
+    fn failure_response_carries_the_status_and_message() {
+        // The degraded arm renders as the 503 JSON error the server always
+        // returned; the message travels through `QueryFailure::message`.
+        let resp = failure_response(&QueryFailure::Unavailable("config failed to load".into()));
+        assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE);
+        assert_eq!(
+            resp.headers().get(header::CONTENT_TYPE).unwrap(),
+            "application/json"
+        );
+    }
+
+    #[test]
+    fn error_response_sets_json_content_type() {
+        let resp = error_response(StatusCode::BAD_REQUEST, "boom");
+        assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
+        assert_eq!(
+            resp.headers().get(header::CONTENT_TYPE).unwrap(),
+            "application/json"
+        );
+    }
+}