dirsql 0.3.8__cp314-cp314-win_amd64.whl

dirsql/__init__.py

@@ -0,0 +1,9 @@
+"""dirsql - Ephemeral SQL index over a local directory.
+
+Also available for Rust (crates.io: ``dirsql``) and TypeScript (npm: ``dirsql``).
+"""
+
+from dirsql._dirsql import Table, RowEvent, __version__
+from dirsql._async import DirSQL
+
+__all__ = ["DirSQL", "Table", "RowEvent", "__version__"]

dirsql/_async.py

@@ -0,0 +1,110 @@
+"""Async-by-default DirSQL wrapper."""
+
+import asyncio
+
+from dirsql._dirsql import DirSQL as _RustDirSQL
+
+
+class _WatchStream:
+    """Async iterator that polls for file events."""
+
+    def __init__(self, db):
+        self._db = db
+        self._started = False
+        self._buffer = []
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        if not self._started:
+            await asyncio.to_thread(self._db._start_watcher)
+            self._started = True
+
+        while True:
+            if self._buffer:
+                return self._buffer.pop(0)
+            events = await asyncio.to_thread(self._db._poll_events, 200)
+            if events:
+                self._buffer.extend(events)
+
+
+class DirSQL:
+    """Async-by-default wrapper around the Rust DirSQL engine.
+
+    Usage:
+        # Programmatic:
+        db = DirSQL(root, tables=[...])
+        # From a config file:
+        db = DirSQL(config="./my-config.toml")
+
+        await db.ready()
+        results = await db.query("SELECT ...")
+        async for event in db.watch():
+            ...
+
+    At least one of ``root`` or ``config`` must be supplied. When both are
+    set, the explicit ``root`` wins over any ``[dirsql].root`` in the config
+    file (a warning is emitted on stderr).
+
+    Pass ``persist=True`` to keep an on-disk SQLite cache (default location:
+    ``<root>/.dirsql/cache.db``). Override the location with ``persist_path``.
+    """
+
+    def __init__(
+        self,
+        root=None,
+        *,
+        tables=None,
+        ignore=None,
+        config=None,
+        persist=False,
+        persist_path=None,
+    ):
+        if root is None and config is None:
+            raise TypeError("DirSQL requires either a root directory or a config= path")
+        self._root = root
+        self._tables = tables
+        self._ignore = ignore
+        self._config = config
+        self._persist = persist
+        self._persist_path = persist_path
+        self._db = None
+        self._ready_event = asyncio.Event()
+        self._init_error = None
+        self._task = asyncio.ensure_future(self._init_bg())
+
+    async def _init_bg(self):
+        """Run the scan in the background."""
+        try:
+            self._db = await asyncio.to_thread(
+                _RustDirSQL,
+                self._root,
+                tables=self._tables,
+                ignore=self._ignore,
+                config=self._config,
+                persist=self._persist,
+                persist_path=self._persist_path,
+            )
+        except Exception as exc:
+            self._init_error = exc
+        finally:
+            self._ready_event.set()
+
+    async def ready(self):
+        """Wait until the initial scan is complete.
+
+        Raises any exception that occurred during init.
+        Can be called multiple times safely.
+        """
+        await self._ready_event.wait()
+        if self._init_error is not None:
+            raise self._init_error
+
+    async def query(self, sql):
+        """Execute a SQL query asynchronously."""
+        return await asyncio.to_thread(self._db.query, sql)
+
+    def watch(self):
+        """Start watching for file changes. Returns an async iterable of RowEvent."""
+        return _WatchStream(self._db)

dirsql/_cli/__init__.py

@@ -0,0 +1,6 @@
+"""CLI launcher package.
+
+The ``dirsql`` console-script entry resolves to
+``dirsql._cli.main:main``. See each submodule for the individual
+functions: :mod:`is_windows`, :mod:`binary_path`, :mod:`main`.
+"""

dirsql/_cli/binary_path.py

@@ -0,0 +1,20 @@
+"""Resolve the bundled Rust binary inside the installed wheel."""
+
+from __future__ import annotations
+
+from importlib.resources import files
+
+from dirsql._cli.is_windows import is_windows
+
+
+def binary_path() -> str:
+    name = "dirsql.exe" if is_windows() else "dirsql"
+    path = files("dirsql").joinpath("_binary", name)
+    if not path.is_file():
+        raise FileNotFoundError(
+            f"bundled `{name}` not found at {path}. The dirsql PyPI wheel "
+            "no longer ships the CLI binary (release-tooling regression "
+            "while putitoutthere wires up bundle_cli). Install the CLI via "
+            "`cargo install dirsql --features cli` or `npx dirsql`."
+        )
+    return str(path)

dirsql/_cli/is_windows.py

@@ -0,0 +1,10 @@
+"""Platform check used by the launcher. Isolated so tests can override it
+without mutating the ``os.name`` global (which upsets pathlib)."""
+
+from __future__ import annotations
+
+import os
+
+
+def is_windows() -> bool:
+    return os.name == "nt"

dirsql/_cli/main.py

@@ -0,0 +1,27 @@
+"""Console-script entry point. Execs the bundled binary on POSIX,
+subprocesses it on Windows."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+
+from dirsql._cli.binary_path import binary_path
+from dirsql._cli.is_windows import is_windows
+
+
+def main(argv: list[str] | None = None) -> int:
+    if argv is None:
+        argv = sys.argv[1:]
+    try:
+        binary = binary_path()
+    except FileNotFoundError as exc:
+        print(f"dirsql: {exc}", file=sys.stderr)
+        return 1
+
+    if is_windows():
+        completed = subprocess.run([binary, *argv])
+        return completed.returncode
+    os.execv(binary, [binary, *argv])
+    return 0  # unreachable

dirsql/test_async.py

@@ -0,0 +1,95 @@
+"""Unit tests for the DirSQL async wrapper."""
+
+from __future__ import annotations
+
+import pytest
+
+from dirsql import _async as async_mod
+
+
+class _FakeRustDirSQL:
+    def __init__(
+        self,
+        root=None,
+        *,
+        tables=None,
+        ignore=None,
+        config=None,
+        persist=False,
+        persist_path=None,
+    ):
+        self.root = root
+        self.tables = tables
+        self.ignore = ignore
+        self.config = config
+        self.persist = persist
+        self.persist_path = persist_path
+        self.query_calls = []
+
+    def query(self, sql):
+        self.query_calls.append(sql)
+        return [{"sql": sql}]
+
+
+class _FakeWatcherDb:
+    def __init__(self, events):
+        self.events = list(events)
+        self.started = 0
+        self.poll_calls = []
+
+    def _start_watcher(self):
+        self.started += 1
+
+    def _poll_events(self, timeout_ms):
+        self.poll_calls.append(timeout_ms)
+        if self.events:
+            return self.events.pop(0)
+        return []
+
+
+def describe_DirSQL_async():
+    def describe_ready_and_query():
+        @pytest.mark.asyncio
+        async def it_uses_the_background_db(monkeypatch):
+            monkeypatch.setattr(async_mod, "_RustDirSQL", _FakeRustDirSQL)
+
+            db = async_mod.DirSQL("/tmp/root", tables=["table-a"], ignore=["**/*.tmp"])
+            await db.ready()
+
+            results = await db.query("SELECT 1")
+
+            assert db._db.root == "/tmp/root"
+            assert db._db.tables == ["table-a"]
+            assert db._db.ignore == ["**/*.tmp"]
+            assert db._db.query_calls == ["SELECT 1"]
+            assert results == [{"sql": "SELECT 1"}]
+
+        @pytest.mark.asyncio
+        async def it_propagates_initialization_errors(monkeypatch):
+            class _BoomDirSQL:
+                def __init__(self, *args, **kwargs):
+                    raise RuntimeError("boom")
+
+            monkeypatch.setattr(async_mod, "_RustDirSQL", _BoomDirSQL)
+
+            db = async_mod.DirSQL("/tmp/root", tables=["table-a"])
+
+            with pytest.raises(RuntimeError, match="boom"):
+                await db.ready()
+
+    def describe_watch_stream():
+        @pytest.mark.asyncio
+        async def it_starts_the_watcher_and_buffers_events():
+            stream = async_mod._WatchStream(
+                _FakeWatcherDb(events=[["event-a", "event-b"]])
+            )
+
+            assert stream.__aiter__() is stream
+
+            first = await stream.__anext__()
+            second = await stream.__anext__()
+
+            assert first == "event-a"
+            assert second == "event-b"
+            assert stream._db.started == 1
+            assert stream._db.poll_calls == [200]

dirsql-0.3.8.dist-info/METADATA

@@ -0,0 +1,216 @@
+Metadata-Version: 2.4
+Name: dirsql
+Version: 0.3.8
+Requires-Dist: pytest>=8 ; extra == 'dev'
+Requires-Dist: pytest-describe>=2 ; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23 ; extra == 'dev'
+Requires-Dist: pytest-cov>=5 ; extra == 'dev'
+Requires-Dist: ruff>=0.4 ; extra == 'dev'
+Requires-Dist: maturin>=1.0 ; extra == 'dev'
+Provides-Extra: dev
+Summary: Ephemeral SQL index over a local directory
+Keywords: sql,filesystem,directory,sqlite,index
+Author: Kevin Scott
+License-Expression: MIT
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+
+# `dirsql` (Python SDK)
+
+Ephemeral SQL index over a local directory. Watches a filesystem, ingests structured files into an in-memory SQLite database, and exposes a SQL query interface. The database is purely in-memory -- the filesystem is always the source of truth.
+
+[Documentation](https://thekevinscott.github.io/dirsql/?lang=python)
+
+## Installation
+
+```bash
+pip install dirsql
+```
+
+Requires Python >= 3.12. Ships as a native extension (Rust via PyO3) -- binary wheels are provided for common platforms.
+
+Each wheel also bundles the `dirsql` HTTP-server CLI as a console script, so `pip install dirsql` also gives you a `dirsql` command on `$PATH`. See the [CLI guide](https://github.com/thekevinscott/dirsql/blob/main/docs/guide/cli.md).
+
+## Publishing (maintainers)
+
+Handled by `.github/workflows/publish.yml` (invoked from `minor-release.yml` / `patch-release.yml`). For each target triple the `build` job `cargo build`s the Rust CLI with `--features cli`, stages the binary into `dirsql/_binary/`, runs `maturin build` (which picks the binary up via the `[tool.maturin] include` rule in `pyproject.toml`), and the wheels + sdist are then trusted-published to PyPI.
+
+## Quick Start
+
+```python
+import asyncio
+import json
+import os
+import tempfile
+from dirsql import DirSQL, Table
+
+async def main():
+    # Create some data files
+    root = tempfile.mkdtemp()
+    os.makedirs(os.path.join(root, "comments", "abc"), exist_ok=True)
+    os.makedirs(os.path.join(root, "comments", "def"), exist_ok=True)
+
+    with open(os.path.join(root, "comments", "abc", "index.jsonl"), "w") as f:
+        f.write(json.dumps({"body": "looks good", "author": "alice"}) + "\n")
+        f.write(json.dumps({"body": "needs work", "author": "bob"}) + "\n")
+
+    with open(os.path.join(root, "comments", "def", "index.jsonl"), "w") as f:
+        f.write(json.dumps({"body": "agreed", "author": "carol"}) + "\n")
+
+    # Define a table: DDL, glob pattern, and an extract function
+    db = DirSQL(
+        root,
+        tables=[
+            Table(
+                ddl="CREATE TABLE comments (id TEXT, body TEXT, author TEXT)",
+                glob="comments/**/index.jsonl",
+                extract=lambda path: [
+                    {
+                        "id": os.path.basename(os.path.dirname(path)),
+                        "body": row["body"],
+                        "author": row["author"],
+                    }
+                    for line in open(path, encoding="utf-8").read().splitlines()
+                    for row in [json.loads(line)]
+                ],
+            ),
+        ],
+    )
+    await db.ready()
+
+    # Query with SQL
+    results = await db.query("SELECT * FROM comments WHERE author = 'alice'")
+    # [{"id": "abc", "body": "looks good", "author": "alice"}]
+
+asyncio.run(main())
+```
+
+## Multiple Tables and Joins
+
+```python
+db = DirSQL(
+    root,
+    tables=[
+        Table(
+            ddl="CREATE TABLE posts (title TEXT, author_id TEXT)",
+            glob="posts/*.json",
+            extract=lambda path: [json.loads(open(path, encoding="utf-8").read())],
+        ),
+        Table(
+            ddl="CREATE TABLE authors (id TEXT, name TEXT)",
+            glob="authors/*.json",
+            extract=lambda path: [json.loads(open(path, encoding="utf-8").read())],
+        ),
+    ],
+)
+await db.ready()
+
+results = await db.query("""
+    SELECT posts.title, authors.name
+    FROM posts JOIN authors ON posts.author_id = authors.id
+""")
+```
+
+## Ignoring Files
+
+Pass `ignore` patterns to skip files during scanning and watching:
+
+```python
+db = DirSQL(
+    root,
+    ignore=["**/drafts/**", "**/.git/**"],
+    tables=[...],
+)
+```
+
+## Watching for Changes
+
+`DirSQL` is async by default. The `watch()` method returns an async iterator of row-level change events.
+
+```python
+import asyncio
+import json
+from dirsql import DirSQL, Table
+
+async def main():
+    db = DirSQL(
+        "/path/to/data",
+        tables=[
+            Table(
+                ddl="CREATE TABLE items (name TEXT)",
+                glob="**/*.json",
+                extract=lambda path: [json.loads(open(path, encoding="utf-8").read())],
+            ),
+        ],
+    )
+    await db.ready()
+
+    # Query
+    results = await db.query("SELECT * FROM items")
+
+    # Watch for file changes (insert/update/delete/error events)
+    async for event in db.watch():
+        print(f"{event.action} on {event.table}: {event.row}")
+        if event.action == "error":
+            print(f"  error: {event.error}")
+
+asyncio.run(main())
+```
+
+## API Reference
+
+### `Table(*, ddl, glob, extract)`
+
+Defines how files map to a SQL table.
+
+- **`ddl`** (`str`): A `CREATE TABLE` statement defining the schema.
+- **`glob`** (`str`): A glob pattern matched against file paths relative to root.
+- **`extract`** (`Callable[[str], list[dict]]`): A function receiving the matched file's absolute filesystem path and returning a list of row dicts. dirsql does not read file contents; a callback that needs the file body reads it itself (e.g. `open(path, encoding="utf-8").read()`). Each dict's keys must match the DDL column names.
+
+### `DirSQL(root=None, *, tables=None, ignore=None, config=None)`
+
+Creates an in-memory SQLite database indexed from the directory at `root`. The constructor is sync and returns immediately; scanning runs in a background thread.
+
+At least one of `root` or `config` must be supplied. When both `root` and `config` are passed (or `config` declares `[dirsql].root`), the explicit `root` wins and a warning is emitted on stderr.
+
+- **`root`** (`str | None`): Path to the directory to index. Optional when `config` supplies one.
+- **`tables`** (`list[Table] | None`): Programmatic table definitions. Appended to any tables in the config file.
+- **`ignore`** (`list[str] | None`): Glob patterns for paths to skip. Appended to any `[dirsql].ignore` patterns in the config file.
+- **`config`** (`str | None`): Optional path to a `.dirsql.toml` file. Its `[[table]]` entries, `[dirsql].ignore`, and optional `[dirsql].root` are merged into the constructor's inputs.
+
+#### `await DirSQL.ready()`
+
+Wait for the initial scan to complete. Idempotent -- safe to call multiple times. Raises any exception that occurred during init.
+
+#### `await DirSQL.query(sql) -> list[dict]`
+
+Execute a SQL query. Returns a list of dicts keyed by column name. Internal tracking columns (`_dirsql_*`) are excluded from results.
+
+#### `DirSQL.watch() -> AsyncIterator[RowEvent]`
+
+Returns an async iterator that yields `RowEvent` objects as files change on disk. Starts the filesystem watcher on first iteration.
+
+### `RowEvent`
+
+Emitted by `watch()` when a file change produces row-level diffs.
+
+- **`table`** (`str`): The affected table name.
+- **`action`** (`str`): One of `"insert"`, `"update"`, `"delete"`, `"error"`.
+- **`row`** (`dict | None`): The new row (for insert/update) or deleted row (for delete).
+- **`old_row`** (`dict | None`): The previous row (for update only).
+- **`error`** (`str | None`): Error message (for error events).
+- **`file_path`** (`str | None`): The relative file path that triggered the event.
+
+## How It Works
+
+The Rust core (`rusqlite` + `notify` + `walkdir`) does the heavy lifting:
+
+1. **Startup scan**: Walks the directory tree, matches files to tables via glob patterns, calls the user-provided `extract` function for each file, and inserts rows into an in-memory SQLite database.
+2. **File watching**: Uses the `notify` crate (inotify on Linux, FSEvents on macOS) to detect file creates, modifications, and deletions.
+3. **Row diffing**: When a file changes, the new rows are diffed against the previous rows for that file, producing granular insert/update/delete events.
+4. **Python bindings**: PyO3 exposes the Rust core as a native Python extension module. The async layer runs blocking operations in a thread pool via `asyncio.to_thread`.
+
+## License
+
+MIT
+

dirsql-0.3.8.dist-info/RECORD

@@ -0,0 +1,14 @@
+dirsql/__init__.py,sha256=R4O-oQ4SvQQJqoBEeovbAGMu82hBii_FVlCvLTxXQHc,300
+dirsql/_async.py,sha256=QHAZ_4y68ApkTlTskbX_pZqxKraFXXPvDHsc19tA2cM,3346
+dirsql/_binary/dirsql.exe,sha256=Uqt7OnRm7UagG-3HqsXdEmtMul4emhBNXsze33Hax_Y,6525440
+dirsql/_cli/__init__.py,sha256=nx8YdJ2hCyUMz5Z840dPK2xaYRKwoQjyHtb_Z3c-Fyw,212
+dirsql/_cli/binary_path.py,sha256=9u34ONnOfxYR_ifNE0vAs5UpUpvlWWlPNRYxDW8wz88,730
+dirsql/_cli/is_windows.py,sha256=Gjue2cpKCd-qvuEDus7haB6riQlVRhQ0FqTLH4A2ht4,253
+dirsql/_cli/main.py,sha256=BEX2NqUdCezftkxjd90WOUn2UgsaQuRgxA3O29rNMyA,713
+dirsql/_dirsql.cp314-win_amd64.pyd,sha256=qUE5p6Fpmjsxhbd7RqQdePzH9bIqzcOgWO11LWhTRis,4731392
+dirsql/test_async.py,sha256=BkTwGXqVcxzBUXY44pNBQ8x6iplxfFtW6mV7MDoHN4U,2784
+dirsql-0.3.8.dist-info/METADATA,sha256=jYOam9dAWX1TspOKpGhWtfh_k3E5UhZ0VI1MA57mEiM,8632
+dirsql-0.3.8.dist-info/WHEEL,sha256=eBGhi3OBjShfkato4dop_2MU-x5MJXrOBEBOPEFTPyw,97
+dirsql-0.3.8.dist-info/entry_points.txt,sha256=JnjTYOopHZyXeheGn31QPXGZ2YhhbwteXdJOL4VB3uQ,47
+dirsql-0.3.8.dist-info/sboms/dirsql-py-ext.cyclonedx.json,sha256=3OflGPGXx25J-L8JtDN0GjBFP4yQdLir0tDe8Drs2qQ,92571
+dirsql-0.3.8.dist-info/RECORD,,

dirsql-0.3.8.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: maturin (1.13.3)
+Root-Is-Purelib: false
+Tag: cp314-cp314-win_amd64

dirsql-0.3.8.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+dirsql=dirsql._cli.main:main