dirsql 0.3.82__cp311-abi3-win_amd64.whl

dirsql/__init__.py

@@ -0,0 +1,9 @@
+"""dirsql - Ephemeral SQL index over a local directory.
+
+Also available for Rust (crates.io: ``dirsql``) and TypeScript (npm: ``dirsql``).
+"""
+
+from dirsql._dirsql import Table, RowEvent, __version__
+from dirsql._async import DirSQL
+
+__all__ = ["DirSQL", "Table", "RowEvent", "__version__"]

dirsql/_async.py

@@ -0,0 +1,184 @@
+"""Async-by-default DirSQL wrapper."""
+
+import asyncio
+import os
+
+from dirsql._dirsql import DirSQL as _RustDirSQL
+from dirsql.resolve_config_extensions import resolve_config_extension_specs
+from dirsql.resolve_extension import resolve_extension_path
+
+
+class _WatchStream:
+    """Async iterator that polls for file events."""
+
+    def __init__(self, owner):
+        self._owner = owner
+        self._db = None
+        self._started = False
+        self._buffer = []
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        if not self._started:
+            await self._owner.ready()
+            db = self._owner._db
+            assert db is not None  # ready() returned, so _init_bg set _db
+            self._db = db
+            await asyncio.to_thread(db._start_watcher)
+            self._started = True
+
+        db = self._db
+        assert db is not None
+        while True:
+            if self._buffer:
+                return self._buffer.pop(0)
+            events = await asyncio.to_thread(db._poll_events, 200)
+            if events:
+                self._buffer.extend(events)
+
+
+class DirSQL:
+    """Async-by-default wrapper around the Rust DirSQL engine.
+
+    Usage:
+        # Programmatic:
+        db = DirSQL(root, tables=[...])
+        # From a config file:
+        db = DirSQL(config="./my-config.toml")
+
+        await db.ready()
+        results = await db.query("SELECT ...")
+        async for event in db.watch():
+            ...
+
+    Supply a ``root``, a ``config`` path, or both. When both are set, the
+    explicit ``root`` wins over any ``[dirsql].root`` in the config file (a
+    warning is emitted on stderr). If neither is given, the initial scan
+    fails with a "no root directory" error raised by the core.
+
+    Pass ``persist=True`` to keep an on-disk SQLite cache (default location:
+    ``<root>/.dirsql/cache.db``). Override the location with ``persist_path``.
+
+    Pass ``extensions`` -- a list of ``{"path": ..., "entrypoint": ...}`` dicts
+    (``entrypoint`` optional) -- to load SQLite extensions onto the connection
+    at startup. Any ``[[dirsql.extension]]`` entries in a ``config`` file are
+    appended after the programmatic ones. A ``path`` (programmatic or
+    config-file) may be a bare **package name**, resolved from the installed
+    package in the runtime env.
+    """
+
+    def __init__(
+        self,
+        root=None,
+        *,
+        tables=None,
+        ignore=None,
+        config=None,
+        persist=False,
+        persist_path=None,
+        extensions=None,
+    ):
+        self._root = root
+        self._tables = tables
+        self._ignore = ignore
+        self._config = config
+        self._persist = persist
+        self._persist_path = persist_path
+        self._extensions = extensions
+        self._db = None
+        self._ready_event = asyncio.Event()
+        self._init_error = None
+        self._task = asyncio.ensure_future(self._init_bg())
+
+    async def _init_bg(self):
+        """Run the scan in the background."""
+        try:
+            self._db = await asyncio.to_thread(self._build_db)
+        except Exception as exc:
+            self._init_error = exc
+        finally:
+            self._ready_event.set()
+
+    def _build_db(self):
+        """Resolve extensions and construct the Rust-backed instance.
+
+        Runs on a worker thread (via ``asyncio.to_thread``): both the
+        package-name resolution and the core's initial scan are blocking.
+
+        When the ``config`` file names an extension by bare package name, the
+        SDK resolves every one of the config's ``[[dirsql.extension]]`` entries
+        itself -- appended after the programmatic ones -- and suppresses the
+        core's own config-extension loading so the entries are not loaded a
+        second time (the core cannot resolve a bare name).
+        """
+        extensions = self._resolved_extensions()
+        suppress = False
+        if self._config is not None:
+            config_extensions = resolve_config_extension_specs(self._config)
+            if config_extensions is not None:
+                extensions = [*(extensions or []), *config_extensions]
+                suppress = True
+        return _RustDirSQL(
+            self._root,
+            tables=self._tables,
+            ignore=self._ignore,
+            config=self._config,
+            persist=self._persist,
+            persist_path=self._persist_path,
+            extensions=extensions,
+            suppress_config_extensions=suppress,
+        )
+
+    def _resolved_extensions(self):
+        """Resolve each programmatic extension's ``path`` to a loadable file.
+
+        A bare package name is resolved to the loadable installed in the
+        runtime env; path-looking values pass through verbatim. Config-file
+        ``[[dirsql.extension]]`` entries are handled by ``_build_db``.
+        """
+        if not self._extensions:
+            return self._extensions
+        return [
+            {
+                "path": resolve_extension_path(
+                    e["path"], base=os.getcwd(), resolve_relative=False
+                ),
+                "entrypoint": e.get("entrypoint"),
+            }
+            for e in self._extensions
+        ]
+
+    async def ready(self):
+        """Wait until the initial scan is complete.
+
+        Raises any exception that occurred during init.
+        Can be called multiple times safely.
+        """
+        await self._ready_event.wait()
+        if self._init_error is not None:
+            raise self._init_error
+
+    async def query(self, sql):
+        """Execute a SQL query asynchronously.
+
+        Awaits :meth:`ready` first, so calling ``query`` before an explicit
+        ``await db.ready()`` waits for the background scan (and re-raises any
+        initialization error) instead of failing on a still-``None`` ``_db``.
+        """
+        await self.ready()
+        db = self._db
+        assert db is not None  # ready() returned, so _init_bg set _db
+        return await asyncio.to_thread(db.query, sql)
+
+    def watch(self):
+        """Start watching for file changes. Returns an async iterable of RowEvent.
+
+        Like :meth:`query`, the returned stream awaits :meth:`ready` on its
+        first iteration before starting the watcher, so calling ``watch``
+        before an explicit ``await db.ready()`` waits for the background scan
+        (and surfaces any initialization error) instead of failing on a
+        still-``None`` ``_db``.
+        """
+        return _WatchStream(self)

dirsql/_dirsql.pyi

@@ -0,0 +1,76 @@
+"""Type stubs for the native PyO3 extension module.
+
+Mirrors the surface defined in ``packages/python/src/lib.rs``. Hand-written
+because pyo3-stub-gen would otherwise demand a build-time hook that the
+maturin / putitoutthere release pipeline does not yet run.
+
+Whenever ``src/lib.rs`` adds, renames, or removes a ``#[pyclass]``,
+``#[pymethods]``, or module-level binding, this file MUST be updated in the
+same PR -- and ``PARITY.md`` is the canonical reminder.
+"""
+
+from collections.abc import Callable
+from os import PathLike
+from typing import Any, NotRequired, TypedDict
+
+from typing_extensions import override
+
+__version__: str
+
+Row = dict[str, Any]
+
+class ExtensionSpec(TypedDict):
+    """A SQLite extension to load at startup: a shared-library ``path`` and an
+    optional ``entrypoint`` init-symbol override. Mirrors a
+    ``[[dirsql.extension]]`` config entry."""
+
+    path: str
+    entrypoint: NotRequired[str]
+
+class Table:
+    """A table definition. Construct via keyword arguments only."""
+
+    ddl: str
+    glob: str
+    strict: bool
+
+    def __init__(
+        self,
+        *,
+        ddl: str,
+        glob: str,
+        extract: Callable[[str], list[Row]],
+        strict: bool = False,
+    ) -> None: ...
+
+class RowEvent:
+    """A row event produced by the watch loop."""
+
+    table: str | None
+    action: str
+    row: Row | None
+    old_row: Row | None
+    error: str | None
+    file_path: str | None
+
+    @override
+    def __repr__(self) -> str: ...
+
+class DirSQL:
+    """Synchronous binding class. ``dirsql._async.DirSQL`` wraps it."""
+
+    def __init__(
+        self,
+        root: str | None = None,
+        *,
+        tables: list[Table] | None = None,
+        ignore: list[str] | None = None,
+        config: str | None = None,
+        persist: bool = False,
+        persist_path: str | PathLike[str] | None = None,
+        extensions: list[ExtensionSpec] | None = None,
+        suppress_config_extensions: bool = False,
+    ) -> None: ...
+    def query(self, sql: str) -> list[Row]: ...
+    def _start_watcher(self) -> None: ...
+    def _poll_events(self, timeout_ms: int) -> list[RowEvent]: ...

dirsql/cli/binary_path.py

@@ -0,0 +1,20 @@
+"""Resolve the bundled Rust binary inside the installed wheel."""
+
+from __future__ import annotations
+
+from importlib.resources import files
+
+from dirsql.cli.is_windows import is_windows
+
+
+def binary_path() -> str:
+    name = "dirsql.exe" if is_windows() else "dirsql"
+    path = files("dirsql").joinpath("_binary", name)
+    if not path.is_file():
+        raise FileNotFoundError(
+            f"bundled `{name}` not found at {path}. The dirsql PyPI wheel "
+            "no longer ships the CLI binary (release-tooling regression "
+            "while putitoutthere wires up bundle_cli). Install the CLI via "
+            "`cargo install dirsql --features cli` or `npx dirsql`."
+        )
+    return str(path)

dirsql/cli/is_windows.py

@@ -0,0 +1,9 @@
+"""Platform check used by the launcher."""
+
+from __future__ import annotations
+
+import os
+
+
+def is_windows() -> bool:
+    return os.name == "nt"

dirsql/cli/main.py

@@ -0,0 +1,38 @@
+"""Console-script entry point. Execs the bundled binary on POSIX,
+subprocesses it on Windows. All argv is forwarded transparently to the
+bundled Rust binary."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+
+from .binary_path import binary_path
+from .is_windows import is_windows
+from .resolve_config_extensions import with_resolved_extensions
+
+
+def main(argv: list[str] | None = None) -> int:
+    if argv is None:
+        argv = sys.argv[1:]
+
+    try:
+        binary = binary_path()
+    except FileNotFoundError as exc:
+        print(f"dirsql: {exc}", file=sys.stderr)
+        return 1
+
+    # Resolve any package-name extensions in a TOML config here (the binary
+    # can't) and pass them as `--extension` flags; a no-op otherwise.
+    try:
+        argv = with_resolved_extensions(argv)
+    except Exception as exc:
+        print(f"dirsql: {exc}", file=sys.stderr)
+        return 1
+
+    if is_windows():
+        completed = subprocess.run([binary, *argv])
+        return completed.returncode
+    os.execv(binary, [binary, *argv])
+    return 0

dirsql/cli/resolve_config_extensions.py

@@ -0,0 +1,55 @@
+"""Launcher-side resolution of a TOML config's ``[[dirsql.extension]]`` entries.
+
+The compiled ``dirsql`` binary loads a config's extensions literally -- it
+has no ``importlib``, so it cannot resolve a bare **package name**. When a
+TOML config names an extension by package name, the shared SDK resolver
+(:mod:`dirsql.resolve_config_extensions`) resolves every one of its
+extensions and this launcher passes the resolved literal paths to the binary
+via repeatable ``--extension`` flags; the binary then loads those and ignores
+the config's own extension entries.
+
+Native-language configs (``.py`` / ``.js`` / ``.mjs`` / ``.cjs``) are untouched:
+the binary dispatches those to ``dirsql interpret``, whose handshake already
+carries resolved paths.
+"""
+
+from __future__ import annotations
+
+from ..resolve_config_extensions import resolve_config_extension_specs
+
+# Config extensions the binary dispatches to `dirsql interpret`; never
+# pre-resolved here (that path resolves via the handshake).
+_NATIVE_SUFFIXES = (".py", ".js", ".mjs", ".cjs")
+
+
+def _config_path_from_argv(argv: list[str]) -> str:
+    """The ``--config`` value (``--config X`` or ``--config=X``), or the default."""
+    for i, a in enumerate(argv):
+        if a == "--config":
+            # A bare trailing `--config` (no following value) yields "".
+            return next(iter(argv[i + 1 :]), "")
+        if a.startswith("--config="):
+            return a[len("--config=") :]
+    return "./.dirsql.toml"
+
+
+def with_resolved_extensions(argv: list[str]) -> list[str]:
+    """Return ``argv`` plus ``--extension`` flags when the TOML config names an
+    extension by package name; otherwise return ``argv`` unchanged. Raises if a
+    package name cannot be resolved (the launcher surfaces a clean error)."""
+    if argv and argv[0] == "init":
+        return argv
+    config_path = _config_path_from_argv(argv)
+    if config_path.endswith(_NATIVE_SUFFIXES):
+        return argv
+    specs = resolve_config_extension_specs(config_path)
+    if specs is None:
+        return argv
+    flags: list[str] = []
+    for spec in specs:
+        entrypoint = spec["entrypoint"]
+        flags.append("--extension")
+        flags.append(
+            f"{spec['path']}::{entrypoint}" if entrypoint is not None else spec["path"]
+        )
+    return [*argv, *flags]

dirsql/resolve_config_extensions.py

@@ -0,0 +1,67 @@
+"""SDK-side resolution of a TOML config's ``[[dirsql.extension]]`` entries.
+
+The Rust core loads a config's extensions literally -- it has no
+``importlib``, so it cannot resolve a bare **package name**. When a TOML
+config names an extension by package name, the SDK resolves every one of its
+extensions here, hands the core the resolved literal paths, and suppresses
+the core's own config-extension loading (``suppress_config_extensions``) so
+the config's entries are not loaded a second time.
+
+Shared by the ``DirSQL`` constructor (``config=`` path) and the CLI launcher
+(which converts the resolved specs into ``--extension`` flags).
+"""
+
+from __future__ import annotations
+
+import os
+import tomllib
+
+from .resolve_extension import is_bare_name, resolve_extension_path
+
+
+def resolve_config_extension_specs(config_path):
+    """Resolve a TOML config's ``[[dirsql.extension]]`` entries to literal paths.
+
+    Returns a list of ``{"path", "entrypoint"}`` dicts -- every entry resolved
+    via :func:`resolve_extension_path` against the config file's parent
+    directory -- when at least one entry's ``path`` is a bare package name.
+    Returns ``None`` when the caller should not intervene: the config is
+    missing, malformed, declares no extensions, or uses only literal paths --
+    leaving the core's own loading (and error reporting) untouched. Raises if
+    a package name cannot be resolved.
+    """
+    if not os.path.isfile(config_path):
+        return None
+    try:
+        with open(config_path, "rb") as f:
+            doc = tomllib.load(f)
+    except (OSError, tomllib.TOMLDecodeError):
+        # Leave a malformed / unreadable config for the core to report.
+        return None
+
+    cfg = doc.get("dirsql") or {}
+    entries = cfg.get("extension")
+    if not isinstance(entries, list):
+        return None
+    # Only intervene when at least one path is a bare package name.
+    if not any(
+        isinstance(e, dict)
+        and isinstance(e.get("path"), str)
+        and is_bare_name(e["path"])
+        for e in entries
+    ):
+        return None
+
+    base = os.path.dirname(os.path.abspath(config_path))
+    specs = []
+    for e in entries:
+        entrypoint = e.get("entrypoint")
+        specs.append(
+            {
+                "path": resolve_extension_path(
+                    e["path"], base=base, resolve_relative=True
+                ),
+                "entrypoint": entrypoint if isinstance(entrypoint, str) else None,
+            }
+        )
+    return specs

dirsql/resolve_extension.py

@@ -0,0 +1,101 @@
+"""Resolve an extension entry's ``path`` to a concrete loadable file.
+
+Resolution is an ordered probe (file-first, then package), so only a bare
+package name reaches the package machinery:
+
+1. **Path-looking** (contains a separator, or ends in ``.so`` / ``.dylib`` /
+   ``.dll`` / ``.pyd``) -- returned as a file path: made absolute against
+   ``base`` when ``resolve_relative`` is set (config-file entries), else
+   verbatim (programmatic entries).
+2. **Bare name** -- a same-named local file under ``base`` *shadows* the
+   package; otherwise the package dir is located via
+   :func:`importlib.util.find_spec` and the current platform's loadable is
+   globbed from inside it. Zero matches and multiple matches are both hard
+   errors -- the caller must disambiguate with a literal path.
+"""
+
+import glob as _glob
+import importlib.util
+import os
+import sys
+
+# Suffixes that mark a value as "already a file path", so package resolution
+# is never attempted.
+_LOADABLE_SUFFIXES = (".so", ".dylib", ".dll", ".pyd")
+
+
+def _platform_patterns():
+    """Loadable-file glob(s) for the current platform."""
+    if sys.platform == "darwin":
+        return ("*.dylib",)
+    if sys.platform == "win32":
+        return ("*.dll", "*.pyd")
+    return ("*.so",)
+
+
+def is_bare_name(path):
+    """True when ``path`` is a bare package name rather than a file path."""
+    if os.sep in path or (os.altsep and os.altsep in path):
+        return False
+    return not path.endswith(_LOADABLE_SUFFIXES)
+
+
+def _resolve_package(name):
+    """Locate ``name``'s package dir and glob its platform loadable file."""
+    try:
+        spec = importlib.util.find_spec(name)
+    except (ImportError, ValueError) as exc:
+        raise ValueError(
+            f"could not resolve extension package {name!r}: {exc}"
+        ) from exc
+    if spec is None:
+        raise ValueError(f"could not resolve extension package {name!r}: not installed")
+
+    dirs = list(spec.submodule_search_locations or [])
+    if not dirs and spec.origin and spec.origin not in ("built-in", "frozen"):
+        dirs.append(os.path.dirname(spec.origin))
+    if not dirs:
+        raise ValueError(
+            f"could not resolve extension package {name!r}: no package directory"
+        )
+
+    patterns = _platform_patterns()
+    matches = set()
+    for d in dirs:
+        for pat in patterns:
+            matches.update(_glob.glob(os.path.join(d, "**", pat), recursive=True))
+    found = sorted(matches)
+
+    try:
+        (single,) = found
+    except ValueError:
+        pat_desc = " / ".join(patterns)
+        if not found:
+            raise ValueError(
+                f"no loadable extension file ({pat_desc}) found in package "
+                f"{name!r} (searched {', '.join(dirs)})"
+            ) from None
+        raise ValueError(
+            f"multiple loadable extension files found in package {name!r}: "
+            f"{', '.join(found)}; disambiguate with a literal path"
+        ) from None
+    return single
+
+
+def resolve_extension_path(path, base, resolve_relative):
+    """Resolve an extension ``path`` to a concrete file.
+
+    ``base`` is the directory a relative path and the bare-name shadow probe
+    resolve against (a config file's parent dir, or the cwd for programmatic
+    entries). ``resolve_relative`` makes a relative path-looking value absolute
+    against ``base`` (config-file semantics); when false it is returned verbatim
+    (programmatic semantics).
+    """
+    if not is_bare_name(path):
+        if resolve_relative and not os.path.isabs(path):
+            return os.path.join(base, path)
+        return path
+    local = os.path.join(base, path)
+    if os.path.isfile(local):
+        return local
+    return _resolve_package(path)