dirsql 0.0.24__tar.gz → 0.0.25__tar.gz

{dirsql-0.0.24 → dirsql-0.0.25}/Cargo.lock

@@ -253,18 +253,12 @@ checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1"
 [[package]]
 name = "dirsql"
 version = "0.1.0"
-dependencies = [
- "dirsql-core",
- "pyo3",
- "regex",
-]
-
-[[package]]
-name = "dirsql-core"
-version = "0.1.0"
 dependencies = [
  "criterion",
  "csv",
+ "futures-channel",
+ "futures-executor",
+ "futures-util",
  "globset",
  "notify",
  "regex",
@@ -274,6 +268,7 @@ dependencies = [
  "serde_yaml",
  "tempfile",
  "thiserror",
+ "tokio",
  "toml",
  "walkdir",
 ]
@@ -282,7 +277,7 @@ dependencies = [
 name = "dirsql-napi"
 version = "0.1.0"
 dependencies = [
- "dirsql-core",
+ "dirsql",
  "napi",
  "napi-build",
  "napi-derive",
@@ -290,18 +285,11 @@ dependencies = [
 ]
 
 [[package]]
-name = "dirsql-sdk"
+name = "dirsql-py-ext"
 version = "0.1.0"
 dependencies = [
- "dirsql-core",
- "futures-channel",
- "futures-core",
- "futures-executor",
- "futures-util",
- "serde_json",
- "tempfile",
- "thiserror",
- "tokio",
+ "dirsql",
+ "pyo3",
 ]
 
 [[package]]

dirsql-0.0.25/Cargo.toml

@@ -0,0 +1,18 @@
+[workspace]
+members = ["packages/python", "packages/rust"]
+resolver = "2"
+
+[workspace.package]
+version = "0.1.0"
+edition = "2024"
+license = "MIT"
+repository = "https://github.com/thekevinscott/dirsql"
+
+[workspace.dependencies]
+dirsql = { path = "packages/rust" }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+regex = "1"
+thiserror = "2"
+tokio = { version = "1", features = ["rt", "sync"] }
+futures-channel = "0.3"

{dirsql-0.0.24 → dirsql-0.0.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.0.24
+Version: 0.0.25
 Requires-Dist: pytest>=8 ; extra == 'dev'
 Requires-Dist: pytest-describe>=2 ; extra == 'dev'
 Requires-Dist: pytest-asyncio>=0.23 ; extra == 'dev'

{dirsql-0.0.24 → dirsql-0.0.25}/packages/python/Cargo.toml

@@ -1,16 +1,17 @@
 [package]
-name = "dirsql"
-version = "0.1.0"
-edition = "2024"
+name = "dirsql-py-ext"
+version.workspace = true
+edition.workspace = true
+publish = false
 readme = "README.md"
 
 [lib]
+name = "dirsql_py_ext"
 crate-type = ["cdylib", "rlib"]
 
 [dependencies]
-dirsql-core = { path = "../core" }
+dirsql.workspace = true
 pyo3 = { version = "0.28", features = ["extension-module"], optional = true }
-regex = "1"
 
 [features]
 default = []

dirsql-0.0.25/packages/python/src/lib.rs

@@ -0,0 +1,315 @@
+//! PyO3 binding for `dirsql`. Intentionally thin: all orchestration lives in
+//! `dirsql::DirSQL`. This layer only:
+//!
+//! - wraps a Python `extract` callable in a Rust closure (acquiring the GIL as
+//!   needed) so it can be handed to [`dirsql::Table`]
+//! - converts row dicts, values, and events between Python and Rust
+//! - forwards `new` / `from_config` / `query` / `_start_watcher` /
+//!   `_poll_events` to the corresponding `DirSQL` methods
+//!
+//! The Python-side async wrapper (`dirsql._async.DirSQL`) drives this binding
+//! via `asyncio.to_thread`.
+
+#[cfg(feature = "extension-module")]
+mod python {
+    use ::dirsql::{DirSQL, Row, RowEvent, Table, Value};
+    use pyo3::exceptions::PyRuntimeError;
+    use pyo3::prelude::*;
+    use pyo3::types::{PyDict, PyList};
+    use std::collections::HashMap;
+    use std::time::Duration;
+
+    // -- Public PyO3 classes ------------------------------------------------
+
+    /// A table definition. Mirrors `dirsql::Table` but holds a Python
+    /// callable for `extract`.
+    #[pyclass(name = "Table", frozen)]
+    struct PyTable {
+        #[pyo3(get)]
+        ddl: String,
+        #[pyo3(get)]
+        glob: String,
+        extract: Py<PyAny>,
+        #[pyo3(get)]
+        strict: bool,
+    }
+
+    #[pymethods]
+    impl PyTable {
+        #[new]
+        #[pyo3(signature = (*, ddl, glob, extract, strict=false))]
+        fn new(ddl: String, glob: String, extract: Py<PyAny>, strict: bool) -> Self {
+            PyTable {
+                ddl,
+                glob,
+                extract,
+                strict,
+            }
+        }
+    }
+
+    /// A row event produced by the watch loop.
+    #[pyclass(name = "RowEvent", frozen)]
+    struct PyRowEvent {
+        #[pyo3(get)]
+        table: String,
+        #[pyo3(get)]
+        action: String,
+        #[pyo3(get)]
+        row: Option<Py<PyDict>>,
+        #[pyo3(get)]
+        old_row: Option<Py<PyDict>>,
+        #[pyo3(get)]
+        error: Option<String>,
+        #[pyo3(get)]
+        file_path: Option<String>,
+    }
+
+    #[pymethods]
+    impl PyRowEvent {
+        fn __repr__(&self) -> String {
+            format!("RowEvent(table={:?}, action={:?})", self.table, self.action)
+        }
+    }
+
+    /// Synchronous binding class. `dirsql._async.DirSQL` wraps it with
+    /// `asyncio.to_thread` to produce the async public API.
+    #[pyclass(name = "DirSQL")]
+    struct PyDirSQL {
+        inner: DirSQL,
+    }
+
+    #[pymethods]
+    impl PyDirSQL {
+        #[new]
+        #[pyo3(signature = (root, *, tables, ignore=None))]
+        fn new(
+            py: Python<'_>,
+            root: String,
+            tables: Vec<PyRef<'_, PyTable>>,
+            ignore: Option<Vec<String>>,
+        ) -> PyResult<Self> {
+            let rust_tables: Vec<Table> = tables.iter().map(|t| build_table(py, t)).collect();
+
+            let inner = py
+                .detach(move || match ignore {
+                    Some(ig) => DirSQL::with_ignore(root, rust_tables, ig),
+                    None => DirSQL::new(root, rust_tables),
+                })
+                .map_err(to_py_err)?;
+
+            Ok(Self { inner })
+        }
+
+        #[classmethod]
+        fn from_config(
+            _cls: &Bound<'_, pyo3::types::PyType>,
+            py: Python<'_>,
+            path: String,
+        ) -> PyResult<Self> {
+            let inner = py
+                .detach(move || DirSQL::from_config_path(&path))
+                .map_err(to_py_err)?;
+            Ok(Self { inner })
+        }
+
+        fn query(&self, py: Python<'_>, sql: String) -> PyResult<Py<PyList>> {
+            let db = self.inner.clone();
+            let rows = py.detach(move || db.query(&sql)).map_err(to_py_err)?;
+
+            let list = PyList::empty(py);
+            for row in rows {
+                list.append(value_row_to_py_dict(py, &row)?)?;
+            }
+            Ok(list.unbind())
+        }
+
+        fn _start_watcher(&self, py: Python<'_>) -> PyResult<()> {
+            let db = self.inner.clone();
+            py.detach(move || db.start_watching()).map_err(to_py_err)
+        }
+
+        fn _poll_events(&self, py: Python<'_>, timeout_ms: u64) -> PyResult<Vec<PyRowEvent>> {
+            let db = self.inner.clone();
+            let events = py
+                .detach(move || db.poll_events(Duration::from_millis(timeout_ms)))
+                .map_err(to_py_err)?;
+
+            events.iter().map(|e| row_event_to_py(py, e)).collect()
+        }
+    }
+
+    // -- Helpers ------------------------------------------------------------
+
+    fn build_table(py: Python<'_>, t: &PyTable) -> Table {
+        let extract_ref = t.extract.clone_ref(py);
+        let mut table = Table::try_new(
+            t.ddl.clone(),
+            t.glob.clone(),
+            make_extract_closure(extract_ref),
+        );
+        table.strict = t.strict;
+        table
+    }
+
+    type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
+
+    fn make_extract_closure(
+        extract: Py<PyAny>,
+    ) -> impl Fn(&str, &str) -> std::result::Result<Vec<Row>, BoxError> + Send + Sync + 'static
+    {
+        move |path: &str, content: &str| {
+            Python::attach(|py| -> std::result::Result<Vec<Row>, BoxError> {
+                let result = extract
+                    .call1(py, (path, content))
+                    .map_err(|e| -> BoxError { Box::new(ExtractError(e.to_string())) })?;
+                let raw: Vec<HashMap<String, Py<PyAny>>> = result
+                    .extract(py)
+                    .map_err(|e: PyErr| -> BoxError { Box::new(ExtractError(e.to_string())) })?;
+
+                let mut rows = Vec::with_capacity(raw.len());
+                for r in &raw {
+                    rows.push(
+                        convert_py_row(py, r)
+                            .map_err(|e| -> BoxError { Box::new(ExtractError(e.to_string())) })?,
+                    );
+                }
+                Ok(rows)
+            })
+        }
+    }
+
+    #[derive(Debug)]
+    struct ExtractError(String);
+    impl std::fmt::Display for ExtractError {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            f.write_str(&self.0)
+        }
+    }
+    impl std::error::Error for ExtractError {}
+
+    fn to_py_err<E: std::fmt::Display>(e: E) -> PyErr {
+        PyRuntimeError::new_err(e.to_string())
+    }
+
+    fn row_event_to_py(py: Python<'_>, event: &RowEvent) -> PyResult<PyRowEvent> {
+        Ok(match event {
+            RowEvent::Insert {
+                table,
+                row,
+                file_path,
+            } => PyRowEvent {
+                table: table.clone(),
+                action: "insert".to_string(),
+                row: Some(value_row_to_py_dict(py, row)?),
+                old_row: None,
+                error: None,
+                file_path: Some(file_path.clone()),
+            },
+            RowEvent::Update {
+                table,
+                old_row,
+                new_row,
+                file_path,
+            } => PyRowEvent {
+                table: table.clone(),
+                action: "update".to_string(),
+                row: Some(value_row_to_py_dict(py, new_row)?),
+                old_row: Some(value_row_to_py_dict(py, old_row)?),
+                error: None,
+                file_path: Some(file_path.clone()),
+            },
+            RowEvent::Delete {
+                table,
+                row,
+                file_path,
+            } => PyRowEvent {
+                table: table.clone(),
+                action: "delete".to_string(),
+                row: Some(value_row_to_py_dict(py, row)?),
+                old_row: None,
+                error: None,
+                file_path: Some(file_path.clone()),
+            },
+            RowEvent::Error { file_path, error } => PyRowEvent {
+                table: String::new(),
+                action: "error".to_string(),
+                row: None,
+                old_row: None,
+                error: Some(error.clone()),
+                file_path: Some(file_path.to_string_lossy().to_string()),
+            },
+        })
+    }
+
+    fn value_row_to_py_dict(py: Python<'_>, row: &Row) -> PyResult<Py<PyDict>> {
+        let dict = PyDict::new(py);
+        for (key, value) in row {
+            dict.set_item(key, value_to_py(py, value))?;
+        }
+        Ok(dict.unbind())
+    }
+
+    fn convert_py_row(
+        py: Python<'_>,
+        py_row: &HashMap<String, Py<PyAny>>,
+    ) -> PyResult<HashMap<String, Value>> {
+        let mut row = HashMap::new();
+        for (key, val) in py_row {
+            row.insert(key.clone(), py_to_value(py, val)?);
+        }
+        Ok(row)
+    }
+
+    fn py_to_value(py: Python<'_>, obj: &Py<PyAny>) -> PyResult<Value> {
+        let bound = obj.bind(py);
+
+        if bound.is_none() {
+            return Ok(Value::Null);
+        }
+
+        // bool must precede int (bool is a subclass of int in Python).
+        if bound.is_instance_of::<pyo3::types::PyBool>() {
+            let b: bool = bound.extract()?;
+            return Ok(Value::Integer(if b { 1 } else { 0 }));
+        }
+
+        if let Ok(i) = bound.extract::<i64>() {
+            return Ok(Value::Integer(i));
+        }
+        if let Ok(f) = bound.extract::<f64>() {
+            return Ok(Value::Real(f));
+        }
+        if let Ok(s) = bound.extract::<String>() {
+            return Ok(Value::Text(s));
+        }
+        if let Ok(b) = bound.extract::<Vec<u8>>() {
+            return Ok(Value::Blob(b));
+        }
+
+        // Fall back to the Python repr.
+        Ok(Value::Text(bound.str()?.to_string()))
+    }
+
+    fn value_to_py(py: Python<'_>, value: &Value) -> Py<PyAny> {
+        match value {
+            Value::Null => py.None(),
+            Value::Integer(i) => i.into_pyobject(py).unwrap().into_any().unbind(),
+            Value::Real(f) => f.into_pyobject(py).unwrap().into_any().unbind(),
+            Value::Text(s) => s.into_pyobject(py).unwrap().into_any().unbind(),
+            Value::Blob(b) => b.into_pyobject(py).unwrap().unbind(),
+        }
+    }
+
+    // -- Module registration ------------------------------------------------
+
+    #[pymodule]
+    #[pyo3(name = "_dirsql")]
+    fn py_dirsql_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
+        m.add("__version__", env!("CARGO_PKG_VERSION"))?;
+        m.add_class::<PyTable>()?;
+        m.add_class::<PyDirSQL>()?;
+        m.add_class::<PyRowEvent>()?;
+        Ok(())
+    }
+}

{dirsql-0.0.24 → dirsql-0.0.25}/packages/python/tests/integration/test_async_dirsql.py

@@ -180,7 +180,7 @@ def describe_DirSQL_async():
             async def collect_events():
                 async for event in db.watch():
                     events.append(event)
-                    if len(events) >= 1:
+                    if event.action == "insert":
                         break
 
             task = asyncio.create_task(collect_events())
@@ -188,9 +188,15 @@ def describe_DirSQL_async():
             # Give the watcher time to start
             await asyncio.sleep(0.3)
 
-            # Create a new file
-            with open(os.path.join(tmp_dir, "new_item.json"), "w") as f:
+            # Create a new file atomically -- write to a sibling tmp path
+            # then rename into place. Without this the watcher can fire on
+            # the empty file between open() and write, producing a spurious
+            # error event ahead of the insert.
+            final = os.path.join(tmp_dir, "new_item.json")
+            tmp = final + ".tmp"
+            with open(tmp, "w") as f:
                 json.dump({"name": "apple"}, f)
+            os.replace(tmp, final)
 
             # Wait for events with timeout
             try:
@@ -198,10 +204,10 @@ def describe_DirSQL_async():
             except asyncio.TimeoutError:
                 pytest.fail("Timed out waiting for watch events")
 
-            assert len(events) >= 1
-            assert events[0].action == "insert"
-            assert events[0].table == "items"
-            assert events[0].row["name"] == "apple"
+            insert = next((e for e in events if e.action == "insert"), None)
+            assert insert is not None, f"no insert event in {events!r}"
+            assert insert.table == "items"
+            assert insert.row["name"] == "apple"
 
         @pytest.mark.asyncio
         async def it_emits_delete_events_for_removed_files(tmp_dir):

{dirsql-0.0.24 → dirsql-0.0.25}/packages/python/tests/integration/test_docs_gaps.py

@@ -305,7 +305,7 @@ def describe_watching_guide_positional_identity_gap():
         assert delete_events, "expected at least one delete event when file shrinks"
         # Docs promise positional identity: the third (idx=2) row should be deleted.
         # The current implementation does a full-replace on shrink instead
-        # (see packages/core/src/differ.rs::diff_rows). That is a doc/impl
+        # (see packages/rust/src/differ.rs::diff_rows). That is a doc/impl
         # divergence surfaced in TESTS_AUDIT.md, not fixed here.
         # What we *can* assert without contradicting either side: among the
         # delete events the dropped row (idx=2, name=row-2) must appear.

{dirsql-0.0.24/packages/core → dirsql-0.0.25/packages/rust}/Cargo.toml

@@ -1,28 +1,33 @@
 [package]
-name = "dirsql-core"
-version = "0.1.0"
-edition = "2024"
+name = "dirsql"
 description = "Ephemeral SQL index over a local directory"
-license = "MIT"
-repository = "https://github.com/thekevinscott/dirsql"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
 readme = "README.md"
 
 [dependencies]
 csv = "1"
+futures-channel.workspace = true
 globset = "0.4"
-regex = "1"
 notify = "7"
+regex.workspace = true
 rusqlite = { version = "0.34", features = ["bundled"] }
-serde = { version = "1", features = ["derive"] }
-serde_json = "1"
+serde.workspace = true
+serde_json.workspace = true
 serde_yaml = "0.9"
-thiserror = "2"
+thiserror.workspace = true
+tokio.workspace = true
 toml = "0.8"
 walkdir = "2"
 
 [dev-dependencies]
 criterion = { version = "0.5", features = ["html_reports"] }
+futures-executor = "0.3"
+futures-util = "0.3"
 tempfile = "3"
+tokio = { version = "1", features = ["rt-multi-thread", "macros", "time"] }
 
 [[bench]]
 name = "db_bench"

dirsql-0.0.25/packages/rust/README.md

@@ -0,0 +1,22 @@
+# dirsql
+
+Rust SDK for `dirsql`. Ephemeral SQL index over a local directory.
+
+## Installation
+
+```bash
+cargo add dirsql
+```
+
+## API
+
+- `Table::new(...)` for infallible extractors
+- `Table::try_new(...)` for fallible extractors
+- `DirSQL::new(...)` for synchronous indexing
+- `DirSQL::with_ignore(...)` for indexing with ignore patterns
+- `DirSQL::query(...)` for SQL queries
+- `DirSQL::watch(...)` for an async row-event stream
+
+## License
+
+MIT

{dirsql-0.0.24/packages/core → dirsql-0.0.25/packages/rust}/benches/db_bench.rs

@@ -1,5 +1,5 @@
 use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
-use dirsql_core::db::{Db, Value};
+use dirsql::db::{Db, Value};
 use std::collections::HashMap;
 
 fn make_row(i: usize) -> HashMap<String, Value> {

{dirsql-0.0.24/packages/core → dirsql-0.0.25/packages/rust}/benches/differ_bench.rs

@@ -1,6 +1,6 @@
 use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
-use dirsql_core::db::Value;
-use dirsql_core::differ::diff;
+use dirsql::db::Value;
+use dirsql::differ::diff;
 use std::collections::HashMap;
 
 fn make_rows(count: usize) -> Vec<HashMap<String, Value>> {

{dirsql-0.0.24/packages/core → dirsql-0.0.25/packages/rust}/benches/matcher_bench.rs

@@ -1,5 +1,5 @@
 use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
-use dirsql_core::matcher::TableMatcher;
+use dirsql::matcher::TableMatcher;
 use std::path::Path;
 
 fn make_matcher(pattern_count: usize) -> TableMatcher {

{dirsql-0.0.24/packages/core → dirsql-0.0.25/packages/rust}/benches/scanner_bench.rs

@@ -1,6 +1,6 @@
 use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
-use dirsql_core::matcher::TableMatcher;
-use dirsql_core::scanner::scan_directory;
+use dirsql::matcher::TableMatcher;
+use dirsql::scanner::scan_directory;
 use std::fs;
 use tempfile::TempDir;