dirsql 0.0.26__cp313-cp313-win_amd64.whl

dirsql/__init__.py

@@ -0,0 +1,6 @@
+"""dirsql - Ephemeral SQL index over a local directory."""
+
+from dirsql._dirsql import Table, RowEvent, __version__
+from dirsql._async import DirSQL
+
+__all__ = ["DirSQL", "Table", "RowEvent", "__version__"]

dirsql/_async.py

@@ -0,0 +1,104 @@
+"""Async-by-default DirSQL wrapper."""
+
+import asyncio
+
+from dirsql._dirsql import DirSQL as _RustDirSQL
+
+
+class _WatchStream:
+    """Async iterator that polls for file events."""
+
+    def __init__(self, db):
+        self._db = db
+        self._started = False
+        self._buffer = []
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        if not self._started:
+            await asyncio.to_thread(self._db._start_watcher)
+            self._started = True
+
+        while True:
+            if self._buffer:
+                return self._buffer.pop(0)
+            events = await asyncio.to_thread(self._db._poll_events, 200)
+            if events:
+                self._buffer.extend(events)
+
+
+class DirSQL:
+    """Async-by-default wrapper around the Rust DirSQL engine.
+
+    Usage:
+        db = DirSQL(root, tables=[...])
+        await db.ready()
+        results = await db.query("SELECT ...")
+        async for event in db.watch():
+            ...
+    """
+
+    def __init__(self, root, *, tables, ignore=None):
+        self._root = root
+        self._tables = tables
+        self._ignore = ignore
+        self._db = None
+        self._ready_event = asyncio.Event()
+        self._init_error = None
+        self._task = asyncio.ensure_future(self._init_bg())
+
+    @classmethod
+    def from_config(cls, path):
+        """Create a DirSQL from a .dirsql.toml config file.
+
+        Returns a DirSQL instance. Call ``await db.ready()`` before querying.
+        """
+        instance = object.__new__(cls)
+        instance._root = None
+        instance._tables = None
+        instance._ignore = None
+        instance._db = None
+        instance._ready_event = asyncio.Event()
+        instance._init_error = None
+        instance._task = asyncio.ensure_future(instance._init_from_config(path))
+        return instance
+
+    async def _init_from_config(self, path):
+        """Run from_config scan in the background."""
+        try:
+            self._db = await asyncio.to_thread(_RustDirSQL.from_config, path)
+        except Exception as exc:
+            self._init_error = exc
+        finally:
+            self._ready_event.set()
+
+    async def _init_bg(self):
+        """Run the scan in the background."""
+        try:
+            self._db = await asyncio.to_thread(
+                _RustDirSQL, self._root, tables=self._tables, ignore=self._ignore
+            )
+        except Exception as exc:
+            self._init_error = exc
+        finally:
+            self._ready_event.set()
+
+    async def ready(self):
+        """Wait until the initial scan is complete.
+
+        Raises any exception that occurred during init.
+        Can be called multiple times safely.
+        """
+        await self._ready_event.wait()
+        if self._init_error is not None:
+            raise self._init_error
+
+    async def query(self, sql):
+        """Execute a SQL query asynchronously."""
+        return await asyncio.to_thread(self._db.query, sql)
+
+    def watch(self):
+        """Start watching for file changes. Returns an async iterable of RowEvent."""
+        return _WatchStream(self._db)

dirsql/test_async.py

@@ -0,0 +1,78 @@
+"""Unit tests for the DirSQL async wrapper."""
+
+import pytest
+
+from dirsql import _async as async_mod
+
+
+class _FakeRustDirSQL:
+    def __init__(self, root, *, tables, ignore=None):
+        self.root = root
+        self.tables = tables
+        self.ignore = ignore
+        self.query_calls = []
+
+    def query(self, sql):
+        self.query_calls.append(sql)
+        return [{"sql": sql}]
+
+
+class _FakeWatcherDb:
+    def __init__(self, events):
+        self.events = list(events)
+        self.started = 0
+        self.poll_calls = []
+
+    def _start_watcher(self):
+        self.started += 1
+
+    def _poll_events(self, timeout_ms):
+        self.poll_calls.append(timeout_ms)
+        if self.events:
+            return self.events.pop(0)
+        return []
+
+
+@pytest.mark.asyncio
+async def test_ready_and_query_use_the_background_db(monkeypatch):
+    monkeypatch.setattr(async_mod, "_RustDirSQL", _FakeRustDirSQL)
+
+    db = async_mod.DirSQL("/tmp/root", tables=["table-a"], ignore=["**/*.tmp"])
+    await db.ready()
+
+    results = await db.query("SELECT 1")
+
+    assert db._db.root == "/tmp/root"
+    assert db._db.tables == ["table-a"]
+    assert db._db.ignore == ["**/*.tmp"]
+    assert db._db.query_calls == ["SELECT 1"]
+    assert results == [{"sql": "SELECT 1"}]
+
+
+@pytest.mark.asyncio
+async def test_ready_propagates_initialization_errors(monkeypatch):
+    class _BoomDirSQL:
+        def __init__(self, *args, **kwargs):
+            raise RuntimeError("boom")
+
+    monkeypatch.setattr(async_mod, "_RustDirSQL", _BoomDirSQL)
+
+    db = async_mod.DirSQL("/tmp/root", tables=["table-a"])
+
+    with pytest.raises(RuntimeError, match="boom"):
+        await db.ready()
+
+
+@pytest.mark.asyncio
+async def test_watch_stream_starts_and_buffers_events():
+    stream = async_mod._WatchStream(_FakeWatcherDb(events=[["event-a", "event-b"]]))
+
+    assert stream.__aiter__() is stream
+
+    first = await stream.__anext__()
+    second = await stream.__anext__()
+
+    assert first == "event-a"
+    assert second == "event-b"
+    assert stream._db.started == 1
+    assert stream._db.poll_calls == [200]

dirsql-0.0.26.dist-info/METADATA

@@ -0,0 +1,209 @@
+Metadata-Version: 2.4
+Name: dirsql
+Version: 0.0.26
+Requires-Dist: pytest>=8 ; extra == 'dev'
+Requires-Dist: pytest-describe>=2 ; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23 ; extra == 'dev'
+Requires-Dist: pytest-cov>=5 ; extra == 'dev'
+Requires-Dist: ruff>=0.4 ; extra == 'dev'
+Requires-Dist: maturin>=1.0 ; extra == 'dev'
+Provides-Extra: dev
+Summary: Ephemeral SQL index over a local directory
+Keywords: sql,filesystem,directory,sqlite,index
+Author: Kevin Scott
+License-Expression: MIT
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+
+# `dirsql` (Python SDK)
+
+Ephemeral SQL index over a local directory. Watches a filesystem, ingests structured files into an in-memory SQLite database, and exposes a SQL query interface. The database is purely in-memory -- the filesystem is always the source of truth.
+
+## Installation
+
+```bash
+pip install dirsql
+```
+
+Requires Python >= 3.12. Ships as a native extension (Rust via PyO3) -- binary wheels are provided for common platforms.
+
+## Quick Start
+
+```python
+import asyncio
+import json
+import os
+import tempfile
+from dirsql import DirSQL, Table
+
+async def main():
+    # Create some data files
+    root = tempfile.mkdtemp()
+    os.makedirs(os.path.join(root, "comments", "abc"), exist_ok=True)
+    os.makedirs(os.path.join(root, "comments", "def"), exist_ok=True)
+
+    with open(os.path.join(root, "comments", "abc", "index.jsonl"), "w") as f:
+        f.write(json.dumps({"body": "looks good", "author": "alice"}) + "\n")
+        f.write(json.dumps({"body": "needs work", "author": "bob"}) + "\n")
+
+    with open(os.path.join(root, "comments", "def", "index.jsonl"), "w") as f:
+        f.write(json.dumps({"body": "agreed", "author": "carol"}) + "\n")
+
+    # Define a table: DDL, glob pattern, and an extract function
+    db = DirSQL(
+        root,
+        tables=[
+            Table(
+                ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)",
+                glob="comments/**/index.jsonl",
+                extract=lambda path, content: [
+                    {
+                        "id": os.path.basename(os.path.dirname(path)),
+                        "body": row["body"],
+                        "author": row["author"],
+                    }
+                    for line in content.splitlines()
+                    for row in [json.loads(line)]
+                ],
+            ),
+        ],
+    )
+    await db.ready()
+
+    # Query with SQL
+    results = await db.query("SELECT * FROM comments WHERE author = 'alice'")
+    # [{"id": "abc", "body": "looks good", "author": "alice"}]
+
+asyncio.run(main())
+```
+
+## Multiple Tables and Joins
+
+```python
+db = DirSQL(
+    root,
+    tables=[
+        Table(
+            ddl="CREATE TABLE posts (title TEXT, author_id TEXT)",
+            glob="posts/*.json",
+            extract=lambda path, content: [json.loads(content)],
+        ),
+        Table(
+            ddl="CREATE TABLE authors (id TEXT, name TEXT)",
+            glob="authors/*.json",
+            extract=lambda path, content: [json.loads(content)],
+        ),
+    ],
+)
+await db.ready()
+
+results = await db.query("""
+    SELECT posts.title, authors.name
+    FROM posts JOIN authors ON posts.author_id = authors.id
+""")
+```
+
+## Ignoring Files
+
+Pass `ignore` patterns to skip files during scanning and watching:
+
+```python
+db = DirSQL(
+    root,
+    ignore=["**/drafts/**", "**/.git/**"],
+    tables=[...],
+)
+```
+
+## Watching for Changes
+
+`DirSQL` is async by default. The `watch()` method returns an async iterator of row-level change events.
+
+```python
+import asyncio
+import json
+from dirsql import DirSQL, Table
+
+async def main():
+    db = DirSQL(
+        "/path/to/data",
+        tables=[
+            Table(
+                ddl="CREATE TABLE items (name TEXT)",
+                glob="**/*.json",
+                extract=lambda path, content: [json.loads(content)],
+            ),
+        ],
+    )
+    await db.ready()
+
+    # Query
+    results = await db.query("SELECT * FROM items")
+
+    # Watch for file changes (insert/update/delete/error events)
+    async for event in db.watch():
+        print(f"{event.action} on {event.table}: {event.row}")
+        if event.action == "error":
+            print(f"  error: {event.error}")
+
+asyncio.run(main())
+```
+
+## API Reference
+
+### `Table(*, ddl, glob, extract)`
+
+Defines how files map to a SQL table.
+
+- **`ddl`** (`str`): A `CREATE TABLE` statement defining the schema.
+- **`glob`** (`str`): A glob pattern matched against file paths relative to root.
+- **`extract`** (`Callable[[str, str], list[dict]]`): A function receiving `(relative_path, file_content)` and returning a list of row dicts. Each dict's keys must match the DDL column names.
+
+### `DirSQL(root, *, tables, ignore=None)`
+
+Creates an in-memory SQLite database indexed from the directory at `root`. The constructor is sync and returns immediately; scanning runs in a background thread.
+
+- **`root`** (`str`): Path to the directory to index.
+- **`tables`** (`list[Table]`): Table definitions.
+- **`ignore`** (`list[str] | None`): Glob patterns for paths to skip.
+
+#### `await DirSQL.ready()`
+
+Wait for the initial scan to complete. Idempotent -- safe to call multiple times. Raises any exception that occurred during init.
+
+#### `await DirSQL.query(sql) -> list[dict]`
+
+Execute a SQL query. Returns a list of dicts keyed by column name. Internal tracking columns (`_dirsql_*`) are excluded from results.
+
+#### `DirSQL.watch() -> AsyncIterator[RowEvent]`
+
+Returns an async iterator that yields `RowEvent` objects as files change on disk. Starts the filesystem watcher on first iteration.
+
+#### `DirSQL.from_config(path) -> DirSQL`
+
+Create a `DirSQL` instance from a `.dirsql.toml` config file. Returns immediately; scanning runs in the background. Call `await db.ready()` before querying.
+
+### `RowEvent`
+
+Emitted by `watch()` when a file change produces row-level diffs.
+
+- **`table`** (`str`): The affected table name.
+- **`action`** (`str`): One of `"insert"`, `"update"`, `"delete"`, `"error"`.
+- **`row`** (`dict | None`): The new row (for insert/update) or deleted row (for delete).
+- **`old_row`** (`dict | None`): The previous row (for update only).
+- **`error`** (`str | None`): Error message (for error events).
+- **`file_path`** (`str | None`): The relative file path that triggered the event.
+
+## How It Works
+
+The Rust core (`rusqlite` + `notify` + `walkdir`) does the heavy lifting:
+
+1. **Startup scan**: Walks the directory tree, matches files to tables via glob patterns, calls the user-provided `extract` function for each file, and inserts rows into an in-memory SQLite database.
+2. **File watching**: Uses the `notify` crate (inotify on Linux, FSEvents on macOS) to detect file creates, modifications, and deletions.
+3. **Row diffing**: When a file changes, the new rows are diffed against the previous rows for that file, producing granular insert/update/delete events.
+4. **Python bindings**: PyO3 exposes the Rust core as a native Python extension module. The async layer runs blocking operations in a thread pool via `asyncio.to_thread`.
+
+## License
+
+MIT
+

dirsql-0.0.26.dist-info/RECORD

@@ -0,0 +1,8 @@
+dirsql/__init__.py,sha256=rL-bI8j4OUx7DY-fG2chvnS7EsrH0-uWLdjYxyhtNJc,213
+dirsql/_async.py,sha256=FOMPeMojhsCqgbyZ3DYdDxkZn7Khrgke_NYIiubn8pE,3209
+dirsql/_dirsql.cp313-win_amd64.pyd,sha256=fao9eetRBHX6uCU0cgCGTotIp0hNzs4k1Gx1yp5FJ5M,4903936
+dirsql/test_async.py,sha256=XuPb9cW6EKU-xVRZ915-_VN-miKjBO0pYZctMDCNrp8,2164
+dirsql-0.0.26.dist-info/METADATA,sha256=xw1P3oonTN7wx358fZvfpgIrRFOKsNq0BrbxUs65_X4,7300
+dirsql-0.0.26.dist-info/WHEEL,sha256=TDIE_VZkwaFwJY0Olvlg1LUe0IwzGBwZhAVXfIDdWJc,97
+dirsql-0.0.26.dist-info/sboms/dirsql-py-ext.cyclonedx.json,sha256=3NNQTWahR01F2sRAy0eLRSyb5u7gkNcTUb60vtZUE9M,89230
+dirsql-0.0.26.dist-info/RECORD,,

dirsql-0.0.26.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: maturin (1.13.1)
+Root-Is-Purelib: false
+Tag: cp313-cp313-win_amd64