dirsql 0.4.33__tar.gz → 0.4.34__tar.gz

{dirsql-0.4.33 → dirsql-0.4.34}/Cargo.lock

@@ -540,7 +540,7 @@ checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8"
 
 [[package]]
 name = "dirsql"
-version = "0.4.33"
+version = "0.4.34"
 dependencies = [
  "assert_cmd",
  "axum",
@@ -583,7 +583,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql-py-ext"
-version = "0.4.33"
+version = "0.4.34"
 dependencies = [
  "dirsql",
  "pyo3",

{dirsql-0.4.33 → dirsql-0.4.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.4.33
+Version: 0.4.34
 Requires-Dist: tomli>=2 ; python_full_version < '3.11'
 Requires-Dist: bin-shim>=0.1
 Summary: Ephemeral SQL index over a local directory

{dirsql-0.4.33 → dirsql-0.4.34}/docs/reference/cli.md

@@ -483,6 +483,24 @@ dirsql: indexed 41231 files in 3m12s
 
 stdout is untouched — it carries the query result and nothing else.
 
+A query gets the same treatment when it calls a
+[worker-backed function](./config.md#dirsql-function). Those cost one round
+trip per row, so a query that calls one over a whole corpus is the slowest
+thing dirsql does:
+
+```
+dirsql: running 9204 worker calls
+```
+
+and, when the query ends:
+
+```
+dirsql: ran 41231 worker calls in 2m41s
+```
+
+A query that calls no worker prints nothing, whatever the setting — the
+reporting only ever speaks when there is something to report.
+
 By default this is **terminal-only, and only for work slow enough to wonder
 about**: a phase that finishes in under half a second prints nothing at all,
 and a run whose stderr is a pipe or a file prints nothing regardless of how

{dirsql-0.4.33/packages/rust → dirsql-0.4.34}/docs/reference/config.md

@@ -133,6 +133,11 @@ process alive for the rest of the invocation, sending it every subsequent
 call — one process total, never one per row or per file. The worker is torn
 down when the invocation ends.
 
+A query paying per-row round trips reports them on stderr while it runs
+(`dirsql: running 9204 worker calls`), on the same terms as every other
+[progress line](./cli.md#progress-reporting): terminal-only by default, silent
+on a pipe, and gone before the result is printed.
+
 The `timeout` bounds each **round-trip call**, not the query: a query that
 calls the function on 10 000 rows is 10 000 individually timed calls. A call
 that times out, or a worker that crashes or closes its pipes, fails the query

{dirsql-0.4.33 → dirsql-0.4.34}/docs/reference/sdk.md

@@ -408,6 +408,10 @@ one summary of what each cost:
 dirsql: indexed 41231 files in 3m12s
 ```
 
+A `query` that calls a worker-backed function is reported the same way, as a
+running count of round trips (`dirsql: running 9204 worker calls`) — the shape
+`dirsql-plugin-embeddings` produces when it embeds a corpus one row at a time.
+
 This is terminal-only by default, and only for a phase that runs longer than
 half a second — a program whose stderr is a pipe, a file, or a log collector
 sees nothing, whatever the scan costs. Set `DIRSQL_PROGRESS=never` to

{dirsql-0.4.33 → dirsql-0.4.34}/packages/python/Cargo.toml

@@ -4,7 +4,7 @@ name = "dirsql-py-ext"
 # pypi/maturin handler can rewrite it via `write-version` before
 # `maturin build`. `pyproject.toml` declares `dynamic = ["version"]`
 # and maturin reads this field. Mirrors `packages/rust/Cargo.toml`.
-version = "0.4.33"
+version = "0.4.34"
 # abi3 note: the `abi3-py310` pyo3 feature (below) builds ONE stable-ABI
 # `cp310-abi3` wheel per platform that loads on every CPython >= 3.10
 # (matching `pyproject.toml`'s `requires-python = ">=3.10"`), instead of

{dirsql-0.4.33/packages/rust → dirsql-0.4.34/packages/python}/docs/reference/cli.md

@@ -483,6 +483,24 @@ dirsql: indexed 41231 files in 3m12s
 
 stdout is untouched — it carries the query result and nothing else.
 
+A query gets the same treatment when it calls a
+[worker-backed function](./config.md#dirsql-function). Those cost one round
+trip per row, so a query that calls one over a whole corpus is the slowest
+thing dirsql does:
+
+```
+dirsql: running 9204 worker calls
+```
+
+and, when the query ends:
+
+```
+dirsql: ran 41231 worker calls in 2m41s
+```
+
+A query that calls no worker prints nothing, whatever the setting — the
+reporting only ever speaks when there is something to report.
+
 By default this is **terminal-only, and only for work slow enough to wonder
 about**: a phase that finishes in under half a second prints nothing at all,
 and a run whose stderr is a pipe or a file prints nothing regardless of how

{dirsql-0.4.33 → dirsql-0.4.34/packages/python}/docs/reference/config.md

@@ -133,6 +133,11 @@ process alive for the rest of the invocation, sending it every subsequent
 call — one process total, never one per row or per file. The worker is torn
 down when the invocation ends.
 
+A query paying per-row round trips reports them on stderr while it runs
+(`dirsql: running 9204 worker calls`), on the same terms as every other
+[progress line](./cli.md#progress-reporting): terminal-only by default, silent
+on a pipe, and gone before the result is printed.
+
 The `timeout` bounds each **round-trip call**, not the query: a query that
 calls the function on 10 000 rows is 10 000 individually timed calls. A call
 that times out, or a worker that crashes or closes its pipes, fails the query

{dirsql-0.4.33/packages/rust → dirsql-0.4.34/packages/python}/docs/reference/sdk.md

@@ -408,6 +408,10 @@ one summary of what each cost:
 dirsql: indexed 41231 files in 3m12s
 ```
 
+A `query` that calls a worker-backed function is reported the same way, as a
+running count of round trips (`dirsql: running 9204 worker calls`) — the shape
+`dirsql-plugin-embeddings` produces when it embeds a corpus one row at a time.
+
 This is terminal-only by default, and only for a phase that runs longer than
 half a second — a program whose stderr is a pipe, a file, or a log collector
 sees nothing, whatever the scan costs. Set `DIRSQL_PROGRESS=never` to

dirsql-0.4.34/packages/python/e2e-attestations/claude-tackle-957-lrm0z6-worker-calls.json

@@ -0,0 +1,7 @@
+{
+  "command": "uv run python -m pytest tests/e2e/ -x -q",
+  "ran_at": 1787263769,
+  "exit_code": 0,
+  "commit": "bced5b72c85b9c50073fd6326c1528326a1bd9d3",
+  "branch": "claude/tackle-957-lrm0z6-worker-calls"
+}

{dirsql-0.4.33 → dirsql-0.4.34}/packages/rust/Cargo.toml

@@ -6,7 +6,7 @@ description = "Ephemeral SQL index over a local directory"
 # version stays for the binding crates (publish = false). The literal
 # intentionally lags the published version: putitoutthere rewrites it to
 # the planned release version at build time and never commits it back.
-version = "0.4.33"
+version = "0.4.34"
 edition.workspace = true
 # Literal `license` (not `license.workspace = true`) because
 # putitoutthere's preflight check reads `[package].license` directly

{dirsql-0.4.33/packages/python → dirsql-0.4.34/packages/rust}/docs/reference/cli.md

@@ -483,6 +483,24 @@ dirsql: indexed 41231 files in 3m12s
 
 stdout is untouched — it carries the query result and nothing else.
 
+A query gets the same treatment when it calls a
+[worker-backed function](./config.md#dirsql-function). Those cost one round
+trip per row, so a query that calls one over a whole corpus is the slowest
+thing dirsql does:
+
+```
+dirsql: running 9204 worker calls
+```
+
+and, when the query ends:
+
+```
+dirsql: ran 41231 worker calls in 2m41s
+```
+
+A query that calls no worker prints nothing, whatever the setting — the
+reporting only ever speaks when there is something to report.
+
 By default this is **terminal-only, and only for work slow enough to wonder
 about**: a phase that finishes in under half a second prints nothing at all,
 and a run whose stderr is a pipe or a file prints nothing regardless of how

{dirsql-0.4.33/packages/python → dirsql-0.4.34/packages/rust}/docs/reference/config.md

@@ -133,6 +133,11 @@ process alive for the rest of the invocation, sending it every subsequent
 call — one process total, never one per row or per file. The worker is torn
 down when the invocation ends.
 
+A query paying per-row round trips reports them on stderr while it runs
+(`dirsql: running 9204 worker calls`), on the same terms as every other
+[progress line](./cli.md#progress-reporting): terminal-only by default, silent
+on a pipe, and gone before the result is printed.
+
 The `timeout` bounds each **round-trip call**, not the query: a query that
 calls the function on 10 000 rows is 10 000 individually timed calls. A call
 that times out, or a worker that crashes or closes its pipes, fails the query

{dirsql-0.4.33/packages/python → dirsql-0.4.34/packages/rust}/docs/reference/sdk.md

@@ -408,6 +408,10 @@ one summary of what each cost:
 dirsql: indexed 41231 files in 3m12s
 ```
 
+A `query` that calls a worker-backed function is reported the same way, as a
+running count of round trips (`dirsql: running 9204 worker calls`) — the shape
+`dirsql-plugin-embeddings` produces when it embeds a corpus one row at a time.
+
 This is terminal-only by default, and only for a phase that runs longer than
 half a second — a program whose stderr is a pipe, a file, or a log collector
 sees nothing, whatever the scan costs. Set `DIRSQL_PROGRESS=never` to

{dirsql-0.4.33 → dirsql-0.4.34}/packages/rust/src/db.rs

@@ -1,8 +1,10 @@
 use rusqlite::Connection;
 use std::collections::{HashMap, HashSet};
 use std::path::{Path, PathBuf};
+use std::sync::Arc;
 use thiserror::Error;
 
+use crate::functions::CallReporter;
 use crate::parsed_vtab;
 use crate::path_table::{self, PathTable, Resolution};
 use crate::scanner;
@@ -264,6 +266,10 @@ pub struct Db {
     /// The persistent cache a parsed path-table reuses across runs. `None` for
     /// an ephemeral index, which has nowhere to cache to.
     path_table_cache: Option<PathBuf>,
+    /// Counts the worker round trips a query pays for. Shared with every
+    /// declared function registered on `conn`; inert until a function is both
+    /// declared and called.
+    calls: Arc<CallReporter>,
 }
 
 /// The skip rules a fresh `Db` starts with.
@@ -299,6 +305,7 @@ impl Db {
             path_table_gitignore: true,
             path_table_parser: None,
             path_table_cache: None,
+            calls: CallReporter::new(),
         })
     }
 
@@ -321,6 +328,7 @@ impl Db {
             path_table_gitignore: true,
             path_table_parser: None,
             path_table_cache: None,
+            calls: CallReporter::new(),
         })
     }
 
@@ -365,6 +373,12 @@ impl Db {
         self.path_table_cache = Some(path);
     }
 
+    /// The reporter every declared function increments. Handed to
+    /// `functions::register_all` at build time.
+    pub(crate) fn call_reporter(&self) -> &Arc<CallReporter> {
+        &self.calls
+    }
+
     /// Borrow the underlying SQLite connection. Internal use only — exposed
     /// to the `persist` module so it can manage the sidecar tables.
     #[doc(hidden)]
@@ -687,6 +701,12 @@ impl Db {
     /// internal writes (`insert_row`, delete-by-file, persist), which never go
     /// through `query()`, are unaffected.
     pub fn query(&self, sql: &str) -> Result<Vec<HashMap<String, Value>>> {
+        // Bracket the whole statement, prepare included: a worker-backed
+        // function called on every row makes this the slowest thing dirsql
+        // does, and the guard is what erases the counter before the caller
+        // prints the rows.
+        let _calls = self.calls.phase();
+
         // Each iteration must register a table no earlier iteration did; a
         // repeat means the fallback is not making progress, so the SQLite
         // error stands. That is what bounds the loop.

{dirsql-0.4.33 → dirsql-0.4.34}/packages/rust/src/functions.rs

@@ -37,6 +37,7 @@ use rusqlite::Connection;
 use rusqlite::functions::FunctionFlags;
 
 use crate::db::Value;
+use crate::progress::{CallProgress, Progress};
 
 /// The per-call timeout when a `[[dirsql.function]]` entry declares no
 /// `timeout` of its own. A round-trip on a persistent worker cannot be
@@ -60,14 +61,20 @@ pub struct ResolvedFunction {
 
 /// Register every resolved function on `conn`, once per accepted arity.
 /// Purely registration — no worker is spawned here.
+///
+/// Every function shares one `calls` reporter: what a user waiting on a query
+/// wants to know is how many round trips it is paying for, not which function
+/// made them.
 pub(crate) fn register_all(
     conn: &Connection,
     functions: &[ResolvedFunction],
+    calls: &Arc<CallReporter>,
 ) -> rusqlite::Result<()> {
     for function in functions {
         let worker = Arc::new(Worker::for_process(function));
         for &arity in &function.args {
             let worker = Arc::clone(&worker);
+            let calls = Arc::clone(calls);
             conn.create_scalar_function(
                 &function.name,
                 i32::from(arity),
@@ -77,6 +84,7 @@ pub(crate) fn register_all(
                     for i in 0..ctx.len() {
                         args.push(Value::from(rusqlite::types::Value::from(ctx.get_raw(i))));
                     }
+                    calls.record();
                     worker
                         .call(&args)
                         .map_err(|message| rusqlite::Error::UserFunctionError(message.into()))
@@ -87,6 +95,78 @@ pub(crate) fn register_all(
     Ok(())
 }
 
+/// Counts the worker round trips a query pays for, and reports them.
+///
+/// A query that calls a declared function on every row spends one round trip
+/// per row -- `dirsql-plugin-embeddings` runs `embed(content)` over the whole
+/// corpus that way -- so the wall time of a single `query()` can be minutes
+/// with nothing on the terminal (dirsql#957).
+///
+/// The count has to be drawn *and erased* by the process that owns stdout, or
+/// the query result lands on top of a leftover line. A worker cannot do that:
+/// it is SIGKILLed on teardown, and teardown happens after the result is
+/// printed anyway (dirsql#1001).
+///
+/// Behind a mutex because SQLite invokes a scalar function on whatever thread
+/// is running the query.
+pub(crate) struct CallReporter {
+    state: Mutex<CallState>,
+}
+
+struct CallState {
+    count: u64,
+    progress: Box<dyn CallProgress>,
+}
+
+impl CallReporter {
+    pub(crate) fn new() -> Arc<Self> {
+        Arc::new(Self::with_progress(Box::new(Progress::worker_calls())))
+    }
+
+    fn with_progress(progress: Box<dyn CallProgress>) -> Self {
+        Self {
+            state: Mutex::new(CallState { count: 0, progress }),
+        }
+    }
+
+    /// Open a phase and return the guard that closes it. A guard rather than
+    /// a paired call so an error mid-query still ends the phase, and so the
+    /// live line is gone before the caller prints anything.
+    pub(crate) fn phase(&self) -> CallPhase<'_> {
+        if let Ok(mut state) = self.state.lock() {
+            state.count = 0;
+            state.progress.restart();
+        }
+        CallPhase(self)
+    }
+
+    /// Record one round trip. A poisoned mutex costs the counter, never the
+    /// call: reporting is not worth failing a query over.
+    fn record(&self) {
+        if let Ok(mut state) = self.state.lock() {
+            state.count += 1;
+            let count = state.count;
+            state.progress.update(count);
+        }
+    }
+
+    fn end(&self) {
+        if let Ok(mut state) = self.state.lock() {
+            let count = state.count;
+            state.progress.finish(count);
+        }
+    }
+}
+
+/// The open phase. Ends when it drops.
+pub(crate) struct CallPhase<'a>(&'a CallReporter);
+
+impl Drop for CallPhase<'_> {
+    fn drop(&mut self) {
+        self.0.end();
+    }
+}
+
 /// The registration flags for a declared function: UTF-8 always, plus
 /// `SQLITE_DETERMINISTIC` when the entry opts in.
 fn function_flags(deterministic: bool) -> FunctionFlags {
@@ -438,6 +518,117 @@ mod tests {
         Value::Text(s.to_string())
     }
 
+    // --- call reporting ----------------------------------------------------
+
+    /// A [`CallProgress`] double recording what it was asked to do, so the
+    /// orchestration is assertable in-module. What the reporter *draws* is
+    /// `progress.rs`'s business and is unit-tested there.
+    #[derive(Clone, Default)]
+    struct Recorder(Arc<Mutex<Vec<String>>>);
+
+    impl Recorder {
+        fn events(&self) -> Vec<String> {
+            self.0.lock().unwrap().clone()
+        }
+
+        fn push(&self, event: String) {
+            self.0.lock().unwrap().push(event);
+        }
+    }
+
+    struct RecordingProgress(Recorder);
+
+    impl CallProgress for RecordingProgress {
+        fn update(&mut self, done: u64) {
+            self.0.push(format!("update {done}"));
+        }
+
+        fn finish(&mut self, done: u64) {
+            self.0.push(format!("finish {done}"));
+        }
+
+        fn restart(&mut self) {
+            self.0.push("restart".to_string());
+        }
+    }
+
+    fn reporter() -> (CallReporter, Recorder) {
+        let recorder = Recorder::default();
+        let reporter = CallReporter::with_progress(Box::new(RecordingProgress(recorder.clone())));
+        (reporter, recorder)
+    }
+
+    #[test]
+    fn a_recorded_round_trip_is_reported_as_a_running_count() {
+        let (reporter, recorder) = reporter();
+
+        reporter.record();
+        reporter.record();
+
+        assert_eq!(recorder.events(), ["update 1", "update 2"]);
+    }
+
+    /// The phase is what the count belongs to: a second query must not carry
+    /// the first one's total, and the reporter has to be told to start over.
+    #[test]
+    fn opening_a_phase_restarts_the_reporter_and_the_count() {
+        let (reporter, recorder) = reporter();
+        reporter.record();
+        reporter.record();
+
+        {
+            let _phase = reporter.phase();
+            reporter.record();
+        }
+
+        assert_eq!(
+            recorder.events(),
+            ["update 1", "update 2", "restart", "update 1", "finish 1"],
+            "the new phase counts from one, not from three"
+        );
+    }
+
+    /// Dropping the guard is what ends the phase — the whole reason it is a
+    /// guard, since a query can leave by an error path.
+    #[test]
+    fn the_phase_guard_finishes_when_it_drops() {
+        let (reporter, recorder) = reporter();
+
+        {
+            let _phase = reporter.phase();
+            reporter.record();
+            reporter.record();
+            assert_eq!(
+                recorder.events(),
+                ["restart", "update 1", "update 2"],
+                "nothing is finished while the phase is open"
+            );
+        }
+
+        assert_eq!(
+            recorder.events(),
+            ["restart", "update 1", "update 2", "finish 2"],
+            "the closed phase reports the count it accumulated"
+        );
+    }
+
+    /// The production constructor wires a real [`Progress`] through the same
+    /// trait, so a whole phase runs against it here — the double proves the
+    /// orchestration, this proves the wiring is connected to something real.
+    /// It draws nothing: the reporter is disabled unless stderr is a terminal.
+    #[test]
+    fn the_production_reporter_counts_round_trips() {
+        let reporter = CallReporter::new();
+
+        {
+            let _phase = reporter.phase();
+            reporter.record();
+            reporter.record();
+        }
+
+        assert_eq!(reporter.state.lock().unwrap().count, 2);
+    }
+
     // --- wire encoding -----------------------------------------------------
 
     #[test]

{dirsql-0.4.33 → dirsql-0.4.34}/packages/rust/src/lib.rs

@@ -885,7 +885,8 @@ impl DirSQL {
         // Register declared functions after extensions so both are available
         // to queries. Registration is inert: no worker process exists until a
         // query's first call to the function.
-        functions::register_all(db.conn(), &functions).map_err(DirSqlError::sqlite)?;
+        functions::register_all(db.conn(), &functions, db.call_reporter())
+            .map_err(DirSqlError::sqlite)?;
 
         // Sweep a cache the reconcile rejected. It happens *here*, after the
         // extensions and functions are on the connection, rather than where