dirsql 0.4.1__cp310-abi3-win_amd64.whl

dirsql/__init__.py

@@ -0,0 +1,9 @@
+"""dirsql - Ephemeral SQL index over a local directory.
+
+Also available for Rust (crates.io: ``dirsql``) and TypeScript (npm: ``dirsql``).
+"""
+
+from dirsql._dirsql import Table, RowEvent, __version__
+from dirsql._async import DirSQL
+
+__all__ = ["DirSQL", "Table", "RowEvent", "__version__"]

dirsql/_async.py

@@ -0,0 +1,194 @@
+"""Async-by-default DirSQL wrapper."""
+
+import asyncio
+import os
+
+from dirsql._dirsql import DirSQL as _RustDirSQL
+from dirsql.resolve_config_extensions import resolve_configs_extension_specs
+from dirsql.resolve_extension import resolve_extension_path
+
+
+class _WatchStream:
+    """Async iterator that polls for file events."""
+
+    def __init__(self, owner):
+        self._owner = owner
+        self._db = None
+        self._started = False
+        self._buffer = []
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        if not self._started:
+            await self._owner.ready()
+            db = self._owner._db
+            assert db is not None  # ready() returned, so _init_bg set _db
+            self._db = db
+            await asyncio.to_thread(db._start_watcher)
+            self._started = True
+
+        db = self._db
+        assert db is not None
+        while True:
+            if self._buffer:
+                return self._buffer.pop(0)
+            events = await asyncio.to_thread(db._poll_events, 200)
+            if events:
+                self._buffer.extend(events)
+
+
+class DirSQL:
+    """Async-by-default wrapper around the Rust DirSQL engine.
+
+    Usage:
+        # Programmatic:
+        db = DirSQL(root, tables=[...])
+        # From a config file:
+        db = DirSQL(config="./my-config.toml")
+
+        await db.ready()
+        results = await db.query("SELECT ...")
+        async for event in db.watch():
+            ...
+
+    The index root is the explicit ``root`` when given, else the process
+    current working directory. A ``config`` file's location never sets the
+    root -- it only supplies tables, ignore patterns, and extensions. There
+    is no ``[dirsql].root`` config key. Constructing with neither ``root``
+    nor ``config`` roots at the cwd (no error is raised).
+
+    Pass ``persist=True`` to keep an on-disk SQLite cache (default location:
+    ``<root>/.dirsql/cache.db``). Override the location with ``persist_path``.
+
+    Pass ``extensions`` -- a list of ``{"path": ..., "entrypoint": ...}`` dicts
+    (``entrypoint`` optional) -- to load SQLite extensions onto the connection
+    at startup. Any ``[[dirsql.extension]]`` entries in a ``config`` file are
+    appended after the programmatic ones. A ``path`` (programmatic or
+    config-file) may be a bare **package name**, resolved from the installed
+    package in the runtime env.
+    """
+
+    def __init__(
+        self,
+        root=None,
+        *,
+        tables=None,
+        ignore=None,
+        config=None,
+        persist=False,
+        persist_path=None,
+        extensions=None,
+    ):
+        self._root = root
+        self._tables = tables
+        self._ignore = ignore
+        self._config = config
+        # A single path or a list of paths; the list merges in order (each
+        # config's [[table]] / ignore / [[dirsql.extension]] accumulate).
+        self._config_paths = (
+            []
+            if config is None
+            else [config]
+            if isinstance(config, str)
+            else list(config)
+        )
+        self._persist = persist
+        self._persist_path = persist_path
+        self._extensions = extensions
+        self._db = None
+        self._ready_event = asyncio.Event()
+        self._init_error = None
+        self._task = asyncio.ensure_future(self._init_bg())
+
+    async def _init_bg(self):
+        """Run the scan in the background."""
+        try:
+            self._db = await asyncio.to_thread(self._build_db)
+        except Exception as exc:
+            self._init_error = exc
+        finally:
+            self._ready_event.set()
+
+    def _build_db(self):
+        """Resolve extensions and construct the Rust-backed instance.
+
+        Runs on a worker thread (via ``asyncio.to_thread``): both the
+        package-name resolution and the core's initial scan are blocking.
+
+        When the ``config`` file names an extension by bare package name, the
+        SDK resolves every one of the config's ``[[dirsql.extension]]`` entries
+        itself -- appended after the programmatic ones -- and suppresses the
+        core's own config-extension loading so the entries are not loaded a
+        second time (the core cannot resolve a bare name).
+        """
+        extensions = self._resolved_extensions()
+        suppress = False
+        if self._config_paths:
+            config_extensions = resolve_configs_extension_specs(self._config_paths)
+            if config_extensions is not None:
+                extensions = [*(extensions or []), *config_extensions]
+                suppress = True
+        return _RustDirSQL(
+            self._root,
+            tables=self._tables,
+            ignore=self._ignore,
+            config=self._config_paths or None,
+            persist=self._persist,
+            persist_path=self._persist_path,
+            extensions=extensions,
+            suppress_config_extensions=suppress,
+        )
+
+    def _resolved_extensions(self):
+        """Resolve each programmatic extension's ``path`` to a loadable file.
+
+        A bare package name is resolved to the loadable installed in the
+        runtime env; path-looking values pass through verbatim. Config-file
+        ``[[dirsql.extension]]`` entries are handled by ``_build_db``.
+        """
+        if not self._extensions:
+            return self._extensions
+        return [
+            {
+                "path": resolve_extension_path(
+                    e["path"], base=os.getcwd(), resolve_relative=False
+                ),
+                "entrypoint": e.get("entrypoint"),
+            }
+            for e in self._extensions
+        ]
+
+    async def ready(self):
+        """Wait until the initial scan is complete.
+
+        Raises any exception that occurred during init.
+        Can be called multiple times safely.
+        """
+        await self._ready_event.wait()
+        if self._init_error is not None:
+            raise self._init_error
+
+    async def query(self, sql):
+        """Execute a SQL query asynchronously.
+
+        Awaits :meth:`ready` first, so calling ``query`` before an explicit
+        ``await db.ready()`` waits for the background scan (and re-raises any
+        initialization error) instead of failing on a still-``None`` ``_db``.
+        """
+        await self.ready()
+        db = self._db
+        assert db is not None  # ready() returned, so _init_bg set _db
+        return await asyncio.to_thread(db.query, sql)
+
+    def watch(self):
+        """Start watching for file changes. Returns an async iterable of RowEvent.
+
+        Like :meth:`query`, the returned stream awaits :meth:`ready` on its
+        first iteration before starting the watcher, so calling ``watch``
+        before an explicit ``await db.ready()`` waits for the background scan
+        (and surfaces any initialization error) instead of failing on a
+        still-``None`` ``_db``.
+        """
+        return _WatchStream(self)

dirsql/_dirsql.pyi

@@ -0,0 +1,76 @@
+"""Type stubs for the native PyO3 extension module.
+
+Mirrors the surface defined in ``packages/python/src/lib.rs``. Hand-written
+because pyo3-stub-gen would otherwise demand a build-time hook that the
+maturin / putitoutthere release pipeline does not yet run.
+
+Whenever ``src/lib.rs`` adds, renames, or removes a ``#[pyclass]``,
+``#[pymethods]``, or module-level binding, this file MUST be updated in the
+same PR -- and ``PARITY.md`` is the canonical reminder.
+"""
+
+from collections.abc import Callable
+from os import PathLike
+from typing import Any, TypedDict
+
+from typing_extensions import NotRequired, override
+
+__version__: str
+
+Row = dict[str, Any]
+
+class ExtensionSpec(TypedDict):
+    """A SQLite extension to load at startup: a shared-library ``path`` and an
+    optional ``entrypoint`` init-symbol override. Mirrors a
+    ``[[dirsql.extension]]`` config entry."""
+
+    path: str
+    entrypoint: NotRequired[str]
+
+class Table:
+    """A table definition. Construct via keyword arguments only."""
+
+    ddl: str
+    glob: str
+    strict: bool
+
+    def __init__(
+        self,
+        *,
+        ddl: str,
+        glob: str,
+        on_file: Callable[[str], list[Row]],
+        strict: bool = False,
+    ) -> None: ...
+
+class RowEvent:
+    """A row event produced by the watch loop."""
+
+    table: str | None
+    action: str
+    row: Row | None
+    old_row: Row | None
+    error: str | None
+    file_path: str | None
+
+    @override
+    def __repr__(self) -> str: ...
+
+class DirSQL:
+    """Synchronous binding class. ``dirsql._async.DirSQL`` wraps it."""
+
+    def __init__(
+        self,
+        root: str | None = None,
+        *,
+        tables: list[Table] | None = None,
+        ignore: list[str] | None = None,
+        config: list[str] | None = None,
+        persist: bool = False,
+        persist_path: str | PathLike[str] | None = None,
+        extensions: list[ExtensionSpec] | None = None,
+        suppress_config_extensions: bool = False,
+    ) -> None: ...
+    def query(self, sql: str) -> list[Row]: ...
+    def _start_watcher(self) -> None: ...
+    def _poll_events(self, timeout_ms: int) -> list[RowEvent]: ...

dirsql/cli/binary_path.py

@@ -0,0 +1,23 @@
+"""Resolve the bundled Rust binary inside the installed wheel."""
+
+from __future__ import annotations
+
+from importlib.resources import files
+
+from dirsql.cli.is_windows import is_windows
+
+
+def binary_path() -> str:
+    name = "dirsql.exe" if is_windows() else "dirsql"
+    # Chained rather than `joinpath("_binary", name)`: multi-segment joinpath
+    # is 3.11+, and `files()` returns a bare `Traversable` for non-filesystem
+    # loaders.
+    path = files("dirsql").joinpath("_binary").joinpath(name)
+    if not path.is_file():
+        raise FileNotFoundError(
+            f"bundled `{name}` not found at {path}. The dirsql PyPI wheel "
+            "no longer ships the CLI binary (release-tooling regression "
+            "while putitoutthere wires up bundle_cli). Install the CLI via "
+            "`cargo install dirsql --features cli` or `npx dirsql`."
+        )
+    return str(path)

dirsql/cli/discover_plugins/discovered_fragments.py

@@ -0,0 +1,18 @@
+"""Discover installed plugins' fragment paths, ordered by entry-point name."""
+
+from __future__ import annotations
+
+from importlib import metadata
+
+from .fragment_path import fragment_path
+
+ENTRY_POINT_GROUP = "dirsql"
+
+
+def discovered_fragments() -> list[str]:
+    """Fragment paths for every installed plugin, ordered by entry-point name
+    (deterministic, so a running server's ``-c`` list is reproducible)."""
+    entry_points = sorted(
+        metadata.entry_points(group=ENTRY_POINT_GROUP), key=lambda ep: ep.name
+    )
+    return [fragment_path(ep.value) for ep in entry_points]

dirsql/cli/discover_plugins/discovery_disabled.py

@@ -0,0 +1,14 @@
+"""Whether plugin discovery is opted out (flag or env var)."""
+
+from __future__ import annotations
+
+import os
+
+NO_PLUGIN_FLAG = "--no-plugin"
+NO_PLUGIN_ENV = "DIRSQL_NO_PLUGIN"
+
+
+def discovery_disabled(argv: list[str]) -> bool:
+    """True when discovery is opted out via ``--no-plugin`` or
+    ``DIRSQL_NO_PLUGIN``."""
+    return NO_PLUGIN_FLAG in argv or bool(os.environ.get(NO_PLUGIN_ENV))

dirsql/cli/discover_plugins/fragment_path.py

@@ -0,0 +1,25 @@
+"""Resolve a plugin module's shipped ``dirsql.toml`` fragment path."""
+
+from __future__ import annotations
+
+from importlib import resources
+
+FRAGMENT_NAME = "dirsql.toml"
+
+
+def fragment_path(module_name: str) -> str:
+    """Absolute path to a plugin module's shipped ``dirsql.toml``. Raises a
+    clear error naming the plugin when the module or the fragment is missing --
+    never a silent skip."""
+    try:
+        fragment = resources.files(module_name).joinpath(FRAGMENT_NAME)
+    except ModuleNotFoundError as exc:
+        raise ValueError(
+            f"dirsql plugin module {module_name!r} is not importable: {exc}"
+        ) from exc
+    if not fragment.is_file():
+        raise ValueError(
+            f"dirsql plugin {module_name!r} ships no {FRAGMENT_NAME} fragment "
+            f"(expected at {fragment})"
+        )
+    return str(fragment)

dirsql/cli/discover_plugins/user_passed_config.py

@@ -0,0 +1,15 @@
+"""Whether the user's argv already names a config file."""
+
+from __future__ import annotations
+
+
+def user_passed_config(argv: list[str]) -> bool:
+    """True when argv already names a ``-c`` / ``--config`` file -- the user's
+    own config is the base, so the baked-in default is not re-added."""
+    for arg in argv:
+        # `--config` naturally fails `startswith("-c")` (it starts with `--`),
+        # so the three clauses are disjoint: bare/attached short `-c`, long
+        # `--config`, and the `--config=<value>` form.
+        if arg == "--config" or arg.startswith("--config=") or arg.startswith("-c"):
+            return True
+    return False

dirsql/cli/discover_plugins/with_discovered_plugins.py

@@ -0,0 +1,48 @@
+"""Rewrite argv to activate installed plugins (the discovery orchestrator).
+
+This is the public entry point of the ``discover_plugins`` package (installed =
+active, CLI only; #363/#529). A plugin is an ordinary Python package that
+declares ``[project.entry-points.dirsql]`` naming its top-level module and ships
+a ``dirsql.toml`` fragment there; when installed alongside ``dirsql``, the
+``pip``/``uvx`` launcher discovers it and injects the fragment as a ``-c`` flag
+plus the hidden ``--include-default`` (#604) when the user gave no ``-c``.
+Opt out via ``--no-plugin`` / ``DIRSQL_NO_PLUGIN=1``. The compiled binary knows
+nothing about plugins, and the SDK never discovers -- only this CLI launcher.
+The helpers each live in their own module (``user_passed_config``,
+``discovery_disabled``, ``fragment_path``, ``discovered_fragments``).
+"""
+
+from __future__ import annotations
+
+from .discovered_fragments import discovered_fragments
+from .discovery_disabled import NO_PLUGIN_FLAG, discovery_disabled
+from .user_passed_config import user_passed_config
+
+
+def with_discovered_plugins(argv: list[str]) -> list[str]:
+    """Return ``argv`` with each installed plugin's fragment appended as ``-c``
+    (plus ``--include-default`` when the user passed no ``-c``). ``--no-plugin``
+    / ``DIRSQL_NO_PLUGIN`` skip discovery, consuming the flag. ``init`` takes no
+    config, so it is left untouched. Raises if a declared plugin is missing its
+    module or fragment (the launcher surfaces a clean error).
+
+    Appending is safe because config flags are subcommand-local (#609): the
+    user's own ``-c`` sits after the ``query`` subcommand (or at top level in
+    server mode), so the injected flags land in the same clap context and
+    accumulate with it -- plugins merge after the user's config, preserving
+    user-first order.
+    """
+    if discovery_disabled(argv):
+        return [a for a in argv if a != NO_PLUGIN_FLAG]
+    if argv and argv[0] == "init":
+        return argv
+    fragments = discovered_fragments()
+    if not fragments:
+        return argv
+    injected: list[str] = []
+    if not user_passed_config(argv):
+        injected.append("--include-default")
+    for fragment in fragments:
+        injected.append("-c")
+        injected.append(fragment)
+    return [*argv, *injected]

dirsql/cli/is_windows.py

@@ -0,0 +1,9 @@
+"""Platform check used by the launcher."""
+
+from __future__ import annotations
+
+import os
+
+
+def is_windows() -> bool:
+    return os.name == "nt"

dirsql/cli/main.py

@@ -0,0 +1,42 @@
+"""Console-script entry point. Execs the bundled binary on POSIX,
+subprocesses it on Windows. All argv is forwarded transparently to the
+bundled Rust binary."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+
+from .binary_path import binary_path
+from .discover_plugins.with_discovered_plugins import with_discovered_plugins
+from .is_windows import is_windows
+from .resolve_config_extensions import with_resolved_extensions
+
+
+def main(argv: list[str] | None = None) -> int:
+    if argv is None:
+        argv = sys.argv[1:]
+
+    try:
+        binary = binary_path()
+    except FileNotFoundError as exc:
+        print(f"dirsql: {exc}", file=sys.stderr)
+        return 1
+
+    # Discover installed plugins (CLI only) and inject their config fragments as
+    # `-c` flags before resolving extensions; then resolve any package-name
+    # extensions in a TOML config here (the binary can't) as `--extension`
+    # flags. Both are no-ops when nothing applies.
+    try:
+        argv = with_discovered_plugins(argv)
+        argv = with_resolved_extensions(argv)
+    except Exception as exc:
+        print(f"dirsql: {exc}", file=sys.stderr)
+        return 1
+
+    if is_windows():
+        completed = subprocess.run([binary, *argv])
+        return completed.returncode
+    os.execv(binary, [binary, *argv])
+    return 0

dirsql/cli/resolve_config_extensions.py

@@ -0,0 +1,55 @@
+"""Launcher-side resolution of a TOML config's ``[[dirsql.extension]]`` entries.
+
+The compiled ``dirsql`` binary loads a config's extensions literally -- it
+has no ``importlib``, so it cannot resolve a bare **package name**. When a
+TOML config names an extension by package name, the shared SDK resolver
+(:mod:`dirsql.resolve_config_extensions`) resolves every one of its
+extensions and this launcher passes the resolved literal paths to the binary
+via repeatable ``--extension`` flags; the binary then loads those and ignores
+the config's own extension entries.
+
+Native-language configs (``.py`` / ``.js`` / ``.mjs`` / ``.cjs``) are untouched:
+the binary dispatches those to ``dirsql interpret``, whose handshake already
+carries resolved paths.
+"""
+
+from __future__ import annotations
+
+from ..resolve_config_extensions import resolve_config_extension_specs
+
+# Config extensions the binary dispatches to `dirsql interpret`; never
+# pre-resolved here (that path resolves via the handshake).
+_NATIVE_SUFFIXES = (".py", ".js", ".mjs", ".cjs")
+
+
+def _config_path_from_argv(argv: list[str]) -> str:
+    """The ``--config`` value (``--config X`` or ``--config=X``), or the default."""
+    for i, a in enumerate(argv):
+        if a == "--config":
+            # A bare trailing `--config` (no following value) yields "".
+            return next(iter(argv[i + 1 :]), "")
+        if a.startswith("--config="):
+            return a[len("--config=") :]
+    return "./.dirsql.toml"
+
+
+def with_resolved_extensions(argv: list[str]) -> list[str]:
+    """Return ``argv`` plus ``--extension`` flags when the TOML config names an
+    extension by package name; otherwise return ``argv`` unchanged. Raises if a
+    package name cannot be resolved (the launcher surfaces a clean error)."""
+    if argv and argv[0] == "init":
+        return argv
+    config_path = _config_path_from_argv(argv)
+    if config_path.endswith(_NATIVE_SUFFIXES):
+        return argv
+    specs = resolve_config_extension_specs(config_path)
+    if specs is None:
+        return argv
+    flags: list[str] = []
+    for spec in specs:
+        entrypoint = spec["entrypoint"]
+        flags.append("--extension")
+        flags.append(
+            f"{spec['path']}::{entrypoint}" if entrypoint is not None else spec["path"]
+        )
+    return [*argv, *flags]

dirsql/resolve_config_extensions.py

@@ -0,0 +1,126 @@
+"""SDK-side resolution of a TOML config's ``[[dirsql.extension]]`` entries.
+
+The Rust core loads a config's extensions literally -- it has no
+``importlib``, so it cannot resolve a bare **package name**. When a TOML
+config names an extension by package name, the SDK resolves every one of its
+extensions here, hands the core the resolved literal paths, and suppresses
+the core's own config-extension loading (``suppress_config_extensions``) so
+the config's entries are not loaded a second time.
+
+Shared by the ``DirSQL`` constructor (``config=`` path) and the CLI launcher
+(which converts the resolved specs into ``--extension`` flags).
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from importlib import import_module
+
+from .resolve_extension import is_bare_name, resolve_extension_path
+
+
+def _load_toml_module():
+    """Return the TOML parser module for the running interpreter.
+
+    ``tomllib`` is stdlib only on 3.11+; on 3.10 the ``tomli`` backport it was
+    derived from (a version-gated dependency) provides the same surface.
+    Imported by name so a unit test can exercise both arms on any
+    interpreter -- a literal ``import tomllib`` is unreachable on 3.10 no
+    matter what ``sys.version_info`` claims.
+    """
+    if sys.version_info >= (3, 11):
+        return import_module("tomllib")
+    return import_module("tomli")
+
+
+_toml = _load_toml_module()
+
+
+def _load_extension_entries(config_path):
+    """Return ``(entries, base_dir)`` for a config's ``[[dirsql.extension]]``.
+
+    ``None`` when the config is missing, unreadable/malformed, or declares no
+    extension array -- the caller should leave such configs to the core.
+    """
+    if not os.path.isfile(config_path):
+        return None
+    try:
+        with open(config_path, "rb") as f:
+            doc = _toml.load(f)
+    except (OSError, _toml.TOMLDecodeError):
+        # Leave a malformed / unreadable config for the core to report.
+        return None
+
+    entries = (doc.get("dirsql") or {}).get("extension")
+    if not isinstance(entries, list):
+        return None
+    return entries, os.path.dirname(os.path.abspath(config_path))
+
+
+def _has_bare_name(entries):
+    return any(
+        isinstance(e, dict)
+        and isinstance(e.get("path"), str)
+        and is_bare_name(e["path"])
+        for e in entries
+    )
+
+
+def _resolve_entries(entries, base):
+    specs = []
+    for e in entries:
+        entrypoint = e.get("entrypoint")
+        specs.append(
+            {
+                "path": resolve_extension_path(
+                    e["path"], base=base, resolve_relative=True
+                ),
+                "entrypoint": entrypoint if isinstance(entrypoint, str) else None,
+            }
+        )
+    return specs
+
+
+def resolve_config_extension_specs(config_path):
+    """Resolve a TOML config's ``[[dirsql.extension]]`` entries to literal paths.
+
+    Returns a list of ``{"path", "entrypoint"}`` dicts -- every entry resolved
+    via :func:`resolve_extension_path` against the config file's parent
+    directory -- when at least one entry's ``path`` is a bare package name.
+    Returns ``None`` when the caller should not intervene: the config is
+    missing, malformed, declares no extensions, or uses only literal paths --
+    leaving the core's own loading (and error reporting) untouched. Raises if
+    a package name cannot be resolved.
+    """
+    loaded = _load_extension_entries(config_path)
+    if loaded is None:
+        return None
+    entries, base = loaded
+    if not _has_bare_name(entries):
+        return None
+    return _resolve_entries(entries, base)
+
+
+def resolve_configs_extension_specs(config_paths):
+    """Resolve the ``[[dirsql.extension]]`` entries of several configs, in order.
+
+    The SDK intervenes for the whole set only when **some** config names an
+    extension by bare package name (the core can resolve neither package names
+    nor -- once globally suppressed -- the literal entries of the other
+    configs). When it intervenes it resolves **every** config's entries, each
+    against that config's own parent directory, concatenated in ``config_paths``
+    order; the caller suppresses the core's config-extension loading and passes
+    the resolved list. Returns ``None`` when no config uses a package name,
+    leaving every config's loading to the core.
+    """
+    loaded = [_load_extension_entries(p) for p in config_paths]
+    if not any(item is not None and _has_bare_name(item[0]) for item in loaded):
+        return None
+    specs = []
+    for item in loaded:
+        if item is None:
+            continue
+        entries, base = item
+        specs.extend(_resolve_entries(entries, base))
+    return specs