dirsql 0.0.17__tar.gz → 0.0.19__tar.gz

{dirsql-0.0.17 → dirsql-0.0.19}/Cargo.toml

@@ -7,6 +7,7 @@ license = "MIT"
 repository = "https://github.com/thekevinscott/dirsql"
 keywords = ["sql", "filesystem", "directory", "sqlite", "index"]
 categories = ["filesystem", "database"]
+readme = "README.md"
 
 [lib]
 crate-type = ["cdylib", "rlib"]

{dirsql-0.0.17 → dirsql-0.0.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.0.17
+Version: 0.0.19
 Requires-Dist: pytest>=8 ; extra == 'dev'
 Requires-Dist: pytest-describe>=2 ; extra == 'dev'
 Requires-Dist: pytest-asyncio>=0.23 ; extra == 'dev'

dirsql-0.0.19/README.md

@@ -0,0 +1,227 @@
+# dirsql
+
+Ephemeral SQL index over a local directory. Watches a filesystem, ingests structured files into an in-memory SQLite database, and exposes a SQL query interface. On shutdown, the database is discarded -- the filesystem remains the source of truth.
+
+## Why
+
+Structured data stored as flat files (JSONL, JSON) is easy to read, write, diff, and version. But querying across many files is slow -- "show me all unresolved comments across 50 documents" requires opening and parsing every file.
+
+dirsql bridges this gap: files remain the source of truth, but you get SQL queries and real-time change events for free.
+
+## Installation
+
+```bash
+pip install dirsql
+```
+
+Rust (library only, no Python bindings):
+
+```bash
+cargo add dirsql
+```
+
+## Quick Start
+
+```python
+import json
+import os
+import tempfile
+from dirsql import DirSQL, Table
+
+# Create some data files
+root = tempfile.mkdtemp()
+os.makedirs(os.path.join(root, "comments", "abc"), exist_ok=True)
+os.makedirs(os.path.join(root, "comments", "def"), exist_ok=True)
+
+with open(os.path.join(root, "comments", "abc", "index.jsonl"), "w") as f:
+    f.write(json.dumps({"body": "looks good", "author": "alice"}) + "\n")
+    f.write(json.dumps({"body": "needs work", "author": "bob"}) + "\n")
+
+with open(os.path.join(root, "comments", "def", "index.jsonl"), "w") as f:
+    f.write(json.dumps({"body": "agreed", "author": "carol"}) + "\n")
+
+# Define a table: DDL, glob pattern, and an extract function
+db = DirSQL(
+    root,
+    tables=[
+        Table(
+            ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)",
+            glob="comments/**/index.jsonl",
+            extract=lambda path, content: [
+                {
+                    "id": os.path.basename(os.path.dirname(path)),
+                    "body": row["body"],
+                    "author": row["author"],
+                }
+                for line in content.splitlines()
+                for row in [json.loads(line)]
+            ],
+        ),
+    ],
+)
+
+# Query with SQL
+results = db.query("SELECT * FROM comments WHERE author = 'alice'")
+# [{"id": "abc", "body": "looks good", "author": "alice"}]
+```
+
+## Multiple Tables and Joins
+
+```python
+db = DirSQL(
+    root,
+    tables=[
+        Table(
+            ddl="CREATE TABLE posts (title TEXT, author_id TEXT)",
+            glob="posts/*.json",
+            extract=lambda path, content: [json.loads(content)],
+        ),
+        Table(
+            ddl="CREATE TABLE authors (id TEXT, name TEXT)",
+            glob="authors/*.json",
+            extract=lambda path, content: [json.loads(content)],
+        ),
+    ],
+)
+
+results = db.query("""
+    SELECT posts.title, authors.name
+    FROM posts JOIN authors ON posts.author_id = authors.id
+""")
+```
+
+## Async API
+
+`AsyncDirSQL` wraps the synchronous API for use with asyncio. Initialization is awaitable, and `watch()` returns an async iterator of row-level change events.
+
+```python
+import asyncio
+import json
+import os
+from dirsql import AsyncDirSQL, Table
+
+async def main():
+    db = AsyncDirSQL(
+        "/path/to/data",
+        tables=[
+            Table(
+                ddl="CREATE TABLE items (name TEXT)",
+                glob="**/*.json",
+                extract=lambda path, content: [json.loads(content)],
+            ),
+        ],
+    )
+    await db.ready()  # wait for initial scan to complete
+
+    # Query works the same way
+    results = await db.query("SELECT * FROM items")
+
+    # Watch for file changes (insert/update/delete/error events)
+    async for event in db.watch():
+        print(f"{event.action} on {event.table}: {event.row}")
+        if event.action == "error":
+            print(f"  error: {event.error}")
+
+asyncio.run(main())
+```
+
+## Ignoring Files
+
+Pass `ignore` patterns to skip files during scanning and watching:
+
+```python
+db = DirSQL(
+    root,
+    ignore=["**/drafts/**", "**/.git/**"],
+    tables=[...],
+)
+```
+
+## API Reference
+
+### `Table(*, ddl, glob, extract)`
+
+Defines how files map to a SQL table.
+
+- **`ddl`** (`str`): A `CREATE TABLE` statement defining the schema.
+- **`glob`** (`str`): A glob pattern matched against file paths relative to root.
+- **`extract`** (`Callable[[str, str], list[dict]]`): A function receiving `(relative_path, file_content)` and returning a list of row dicts. Each dict's keys must match the DDL column names.
+
+### `DirSQL(root, *, tables, ignore=None)`
+
+Creates an in-memory SQLite database indexed from the directory at `root`.
+
+- **`root`** (`str`): Path to the directory to index.
+- **`tables`** (`list[Table]`): Table definitions.
+- **`ignore`** (`list[str] | None`): Glob patterns for paths to skip.
+
+#### `DirSQL.query(sql) -> list[dict]`
+
+Execute a SQL query. Returns a list of dicts keyed by column name. Internal tracking columns (`_dirsql_*`) are excluded from results.
+
+### `AsyncDirSQL(root, *, tables, ignore=None)`
+
+Async wrapper. Constructor is sync (returns immediately). Call `await db.ready()` to wait for the initial scan.
+
+#### `await AsyncDirSQL.ready()`
+
+Wait for the initial scan to complete. Idempotent -- safe to call multiple times. Raises any exception that occurred during init.
+
+#### `await AsyncDirSQL.query(sql) -> list[dict]`
+
+Same as `DirSQL.query`, but async.
+
+#### `AsyncDirSQL.watch() -> AsyncIterator[RowEvent]`
+
+Returns an async iterator that yields `RowEvent` objects as files change on disk. Starts the filesystem watcher on first iteration.
+
+### `RowEvent`
+
+Emitted by `watch()` when a file change produces row-level diffs.
+
+- **`table`** (`str`): The affected table name.
+- **`action`** (`str`): One of `"insert"`, `"update"`, `"delete"`, `"error"`.
+- **`row`** (`dict | None`): The new row (for insert/update) or deleted row (for delete).
+- **`old_row`** (`dict | None`): The previous row (for update only).
+- **`error`** (`str | None`): Error message (for error events).
+- **`file_path`** (`str | None`): The relative file path that triggered the event.
+
+## How It Works
+
+The Rust core (`rusqlite` + `notify` + `walkdir`) does the heavy lifting:
+
+1. **Startup scan**: Walks the directory tree, matches files to tables via glob patterns, calls the user-provided `extract` function for each file, and inserts rows into an in-memory SQLite database.
+2. **File watching**: Uses the `notify` crate (inotify on Linux, FSEvents on macOS) to detect file creates, modifications, and deletions.
+3. **Row diffing**: When a file changes, the new rows are diffed against the previous rows for that file, producing granular insert/update/delete events.
+4. **Python bindings**: PyO3 exposes the Rust core as a native Python extension module. The async layer runs blocking operations in a thread pool via `asyncio.to_thread`.
+
+The SQLite database is purely ephemeral -- it exists only in memory and is discarded when the `DirSQL` instance is garbage collected. The filesystem is always the source of truth.
+
+## Development
+
+### Prerequisites
+
+- Rust (stable)
+- Python >= 3.12
+- [maturin](https://github.com/PyO3/maturin) for building the Python extension
+- [just](https://github.com/casey/just) as a task runner
+
+### Build and Test
+
+```bash
+# Build the Python extension (dev mode)
+maturin develop
+
+# Run all CI checks
+just ci
+
+# Individual targets
+just test-rust        # Rust unit tests
+just test-integration # Python integration tests
+just clippy           # Rust lints
+just lint             # Python lints (ruff)
+```
+
+## License
+
+MIT

{dirsql-0.0.17 → dirsql-0.0.19}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "dirsql"
-version = "0.0.17"
+version = "0.0.19"
 description = "Ephemeral SQL index over a local directory"
 license = "MIT"
 requires-python = ">=3.12"
@@ -24,7 +24,7 @@ dev = [
 ]
 
 [tool.maturin]
-features = ["pyo3/extension-module"]
+features = ["extension-module"]
 exclude = [
     ".github/",
     ".claude/",

{dirsql-0.0.17 → dirsql-0.0.19}/python/dirsql/_async.py

@@ -33,7 +33,8 @@ class AsyncDirSQL:
     """Async wrapper around DirSQL.
 
     Usage:
-        db = await AsyncDirSQL(root, tables=[...])
+        db = AsyncDirSQL(root, tables=[...])
+        await db.ready()
         results = await db.query("SELECT ...")
         async for event in db.watch():
             ...
@@ -44,15 +45,30 @@ class AsyncDirSQL:
         self._tables = tables
         self._ignore = ignore
         self._db = None
-
-    def __await__(self):
-        return self._init().__await__()
-
-    async def _init(self):
-        self._db = await asyncio.to_thread(
-            DirSQL, self._root, tables=self._tables, ignore=self._ignore
-        )
-        return self
+        self._ready_event = asyncio.Event()
+        self._init_error = None
+        self._task = asyncio.ensure_future(self._init_bg())
+
+    async def _init_bg(self):
+        """Run the scan in the background."""
+        try:
+            self._db = await asyncio.to_thread(
+                DirSQL, self._root, tables=self._tables, ignore=self._ignore
+            )
+        except Exception as exc:
+            self._init_error = exc
+        finally:
+            self._ready_event.set()
+
+    async def ready(self):
+        """Wait until the initial scan is complete.
+
+        Raises any exception that occurred during init.
+        Can be called multiple times safely.
+        """
+        await self._ready_event.wait()
+        if self._init_error is not None:
+            raise self._init_error
 
     async def query(self, sql):
         """Execute a SQL query asynchronously."""

{dirsql-0.0.17 → dirsql-0.0.19}/tests/integration/test_async_dirsql.py

@@ -12,9 +12,9 @@ from dirsql import AsyncDirSQL, Table
 def describe_AsyncDirSQL():
     def describe_init():
         @pytest.mark.asyncio
-        async def it_creates_instance_with_await(jsonl_dir):
-            """AsyncDirSQL can be initialized with await."""
-            db = await AsyncDirSQL(
+        async def it_creates_instance_synchronously(jsonl_dir):
+            """AsyncDirSQL constructor is sync and returns immediately."""
+            db = AsyncDirSQL(
                 jsonl_dir,
                 tables=[
                     Table(
@@ -35,9 +35,9 @@ def describe_AsyncDirSQL():
             assert db is not None
 
         @pytest.mark.asyncio
-        async def it_indexes_files_on_init(jsonl_dir):
-            """Async init scans and indexes directory contents."""
-            db = await AsyncDirSQL(
+        async def it_indexes_files_after_ready(jsonl_dir):
+            """Data is available after awaiting ready()."""
+            db = AsyncDirSQL(
                 jsonl_dir,
                 tables=[
                     Table(
@@ -55,33 +55,61 @@ def describe_AsyncDirSQL():
                     ),
                 ],
             )
+            await db.ready()
             results = await db.query("SELECT * FROM comments")
             assert len(results) == 3
 
         @pytest.mark.asyncio
-        async def it_raises_on_extract_error_during_init(tmp_dir):
-            """Extract lambda errors during init raise exceptions."""
+        async def it_raises_on_extract_error_during_ready(tmp_dir):
+            """Extract lambda errors during ready() raise exceptions."""
             os.makedirs(os.path.join(tmp_dir, "data"), exist_ok=True)
             with open(os.path.join(tmp_dir, "data", "bad.json"), "w") as f:
                 f.write("not valid json")
 
+            db = AsyncDirSQL(
+                tmp_dir,
+                tables=[
+                    Table(
+                        ddl="CREATE TABLE items (name TEXT)",
+                        glob="data/*.json",
+                        extract=lambda path, content: [json.loads(content)],
+                    ),
+                ],
+            )
             with pytest.raises(Exception):
-                await AsyncDirSQL(
-                    tmp_dir,
-                    tables=[
-                        Table(
-                            ddl="CREATE TABLE items (name TEXT)",
-                            glob="data/*.json",
-                            extract=lambda path, content: [json.loads(content)],
-                        ),
-                    ],
-                )
+                await db.ready()
+
+        @pytest.mark.asyncio
+        async def it_allows_multiple_ready_calls(jsonl_dir):
+            """Calling ready() multiple times is safe and idempotent."""
+            db = AsyncDirSQL(
+                jsonl_dir,
+                tables=[
+                    Table(
+                        ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)",
+                        glob="comments/**/index.jsonl",
+                        extract=lambda path, content: [
+                            {
+                                "id": os.path.basename(os.path.dirname(path)),
+                                "body": row["body"],
+                                "author": row["author"],
+                            }
+                            for line in content.splitlines()
+                            for row in [json.loads(line)]
+                        ],
+                    ),
+                ],
+            )
+            await db.ready()
+            await db.ready()
+            results = await db.query("SELECT * FROM comments")
+            assert len(results) == 3
 
     def describe_query():
         @pytest.mark.asyncio
         async def it_returns_results_as_list_of_dicts(jsonl_dir):
             """Async query returns list of dicts with column names."""
-            db = await AsyncDirSQL(
+            db = AsyncDirSQL(
                 jsonl_dir,
                 tables=[
                     Table(
@@ -99,6 +127,7 @@ def describe_AsyncDirSQL():
                     ),
                 ],
             )
+            await db.ready()
             results = await db.query(
                 "SELECT author FROM comments WHERE body = 'first comment'"
             )
@@ -108,7 +137,7 @@ def describe_AsyncDirSQL():
         @pytest.mark.asyncio
         async def it_raises_on_invalid_sql(jsonl_dir):
             """Invalid SQL raises an exception."""
-            db = await AsyncDirSQL(
+            db = AsyncDirSQL(
                 jsonl_dir,
                 tables=[
                     Table(
@@ -126,6 +155,7 @@ def describe_AsyncDirSQL():
                     ),
                 ],
             )
+            await db.ready()
             with pytest.raises(Exception):
                 await db.query("NOT VALID SQL")
 
@@ -133,7 +163,7 @@ def describe_AsyncDirSQL():
         @pytest.mark.asyncio
         async def it_emits_insert_events_for_new_files(tmp_dir):
             """watch() yields insert events when a new file is created."""
-            db = await AsyncDirSQL(
+            db = AsyncDirSQL(
                 tmp_dir,
                 tables=[
                     Table(
@@ -143,6 +173,7 @@ def describe_AsyncDirSQL():
                     ),
                 ],
             )
+            await db.ready()
 
             events = []
 
@@ -179,7 +210,7 @@ def describe_AsyncDirSQL():
             with open(os.path.join(tmp_dir, "doomed.json"), "w") as f:
                 json.dump({"name": "doomed"}, f)
 
-            db = await AsyncDirSQL(
+            db = AsyncDirSQL(
                 tmp_dir,
                 tables=[
                     Table(
@@ -189,6 +220,7 @@ def describe_AsyncDirSQL():
                     ),
                 ],
             )
+            await db.ready()
 
             # Confirm initial data
             results = await db.query("SELECT * FROM items")
@@ -228,7 +260,7 @@ def describe_AsyncDirSQL():
             with open(os.path.join(tmp_dir, "item.json"), "w") as f:
                 json.dump({"name": "draft"}, f)
 
-            db = await AsyncDirSQL(
+            db = AsyncDirSQL(
                 tmp_dir,
                 tables=[
                     Table(
@@ -238,6 +270,7 @@ def describe_AsyncDirSQL():
                     ),
                 ],
             )
+            await db.ready()
 
             events = []
 
@@ -267,7 +300,7 @@ def describe_AsyncDirSQL():
         @pytest.mark.asyncio
         async def it_emits_error_events_for_bad_extract(tmp_dir):
             """watch() yields error events when extract lambda fails."""
-            db = await AsyncDirSQL(
+            db = AsyncDirSQL(
                 tmp_dir,
                 tables=[
                     Table(
@@ -277,6 +310,7 @@ def describe_AsyncDirSQL():
                     ),
                 ],
             )
+            await db.ready()
 
             events = []
 
@@ -305,7 +339,7 @@ def describe_AsyncDirSQL():
         @pytest.mark.asyncio
         async def it_updates_db_on_file_changes(tmp_dir):
             """The database is kept in sync with file system changes."""
-            db = await AsyncDirSQL(
+            db = AsyncDirSQL(
                 tmp_dir,
                 tables=[
                     Table(
@@ -315,6 +349,7 @@ def describe_AsyncDirSQL():
                     ),
                 ],
             )
+            await db.ready()
 
             # Initially empty
             results = await db.query("SELECT * FROM items")