dirsql-plugin-embeddings 0.1.0__tar.gz

dirsql_plugin_embeddings-0.1.0/.gitignore

@@ -0,0 +1,58 @@
+/target
+packages/*/target
+.beads
+.claude/settings.local.json
+.claude/scheduled_tasks.lock
+.worktrees/
+.claude/worktrees/
+
+# Generated by .claude/hooks/select-environment.sh on session start.
+# The environment source files live in agents/environments/ and are checked in.
+agents/build/
+
+# Node
+node_modules/
+*.node
+
+# TypeScript build output
+packages/ts/dist/
+# TypeScript coverage output
+packages/ts/coverage/
+# napi-rs generated type definitions (authored types live in packages/ts/src/)
+packages/ts/index.d.ts
+
+# Python
+__pycache__/
+*.pyc
+*.pyo
+*.so
+.venv/
+.coverage
+.pytest_cache/
+# Rust CLI binary bundled into the wheel by CI (not in source control).
+packages/python/dirsql/_binary/
+
+# putitoutthere per-(mode, triple) staging outputs
+packages/ts/build/
+
+# wireit cache (per-package)
+packages/*/.wireit/
+
+# uv
+uv.lock
+
+# Dolt database files (added by bd init)
+.dolt/
+*.db
+
+# VitePress / docs
+docs/node_modules/
+docs/.vitepress/cache/
+docs/.vitepress/dist/
+
+# Playwright
+docs/test-results/
+docs/playwright-report/
+docs/blob-report/
+docs/.playwright/
+actionlint

dirsql_plugin_embeddings-0.1.0/PKG-INFO

@@ -0,0 +1,5 @@
+Metadata-Version: 2.4
+Name: dirsql-plugin-embeddings
+Version: 0.1.0
+Summary: First-party dirsql plugin: semantic search via an OpenAI-compatible embeddings endpoint.
+Requires-Python: >=3.11

dirsql_plugin_embeddings-0.1.0/README.md

@@ -0,0 +1,58 @@
+# dirsql-plugin-embeddings
+
+A first-party [`dirsql`](https://github.com/thekevinscott/dirsql) plugin that
+adds **semantic search** over a directory of Markdown files. It is the worked
+implementation behind the [Search documents by
+meaning](../../docs/howto/search-by-meaning.md) how-to, swapping that guide's
+local `model2vec` model for any OpenAI-compatible `/v1/embeddings` endpoint.
+
+This is an **in-repo, repo-only** package (not published to an index). It exists
+to prove the dirsql plugin conventions (#531, part of #363) and power the
+semantic-search demo. Deliberately minimal (v0.1): one embedding provider shape,
+one table, no chunking, no config surface beyond three environment variables.
+
+## How it works
+
+The plugin ships a `dirsql.toml` fragment that dirsql discovers when the package
+is installed alongside it. The fragment declares:
+
+- the [`sqlite-vec`](https://github.com/asg017/sqlite-vec) extension, for
+  `vec_distance_cosine()`;
+- a `documents` table whose `on-file` hook embeds each `**/*.md` file into a
+  TEXT `embedding` column;
+- a `pre-query` hook that embeds the incoming question and emits the
+  nearest-neighbor SQL.
+
+Both hooks are console scripts that call the same embedder.
+
+## Configuration
+
+The embedder reads three environment variables (point them at any hosted or
+self-managed OpenAI-compatible inference server):
+
+| Variable | Meaning |
+|---|---|
+| `DIRSQL_EMBEDDINGS_BASE_URL` | Base URL; `/v1/embeddings` is appended. |
+| `DIRSQL_EMBEDDINGS_MODEL` | Model name sent in the request. |
+| `DIRSQL_EMBEDDINGS_API_KEY` | Bearer token for `Authorization`. |
+
+## Console scripts
+
+| Script | Hook | Input | Output |
+|---|---|---|---|
+| `dirsql-embeddings-on-file` | `on-file` | a file's absolute path (`argv[1]`) | one-line JSON row array with `path`, `text`, `embedding` |
+| `dirsql-embeddings-pre-query` | `pre-query` | a raw request body (`argv[1]`) | nearest-neighbor SQL over `documents` |
+
+`pre-query` accepts both a verbatim server body (`{"q": ...}`) and the CLI
+`query` subcommand's `{"sql": <arg>}` wrapper, so `dirsql query '{"q": ...}'` and
+a real `POST /query` both work.
+
+## Tests
+
+Three tiers, per the dirsql testing conventions:
+
+- **unit** (colocated, mocked seams) — `src/dirsql_plugin_embeddings/*_test.py`
+- **integration** (`tests/integration/`) — each console script as a real
+  subprocess against a local stub `/v1/embeddings` server
+- **e2e** (`tests/e2e/`) — the full loop through the real launcher + `dirsql`
+  binary + `sqlite-vec`, nothing mocked but the embedding endpoint

dirsql_plugin_embeddings-0.1.0/e2e-attestations/claude-open-issues-review-le8hm5-531.json

@@ -0,0 +1,7 @@
+{
+  "command": "uv run --with sqlite-vec --with-editable . --with-editable ../../packages/python python -m pytest tests/e2e -x -q",
+  "ran_at": 1784022840,
+  "exit_code": 0,
+  "commit": "5f6caffd4996dfdf09cf798c1deb55f6e216fde5",
+  "branch": "claude/open-issues-review-le8hm5-531"
+}

dirsql_plugin_embeddings-0.1.0/pyproject.toml

@@ -0,0 +1,66 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+# First-party dirsql plugin, published independently to PyPI: the worked
+# implementation behind docs/howto/search-by-meaning.md, backed by an
+# OpenAI-compatible /v1/embeddings endpoint. Exists to prove the plugin
+# rails (#531, part of #363) and power the semantic-search demo.
+name = "dirsql-plugin-embeddings"
+dynamic = ["version"]
+description = "First-party dirsql plugin: semantic search via an OpenAI-compatible embeddings endpoint."
+requires-python = ">=3.11"
+
+[tool.hatch.version]
+source = "vcs"
+# putitoutthere tags this package independently as
+# `dirsql-plugin-embeddings-v<version>`, distinct from the workspace
+# `dirsql-{rust,py,npm}` tags sharing the same git history.
+tag-pattern = "^dirsql-plugin-embeddings-v(?P<version>[0-9]+\\.[0-9]+\\.[0-9]+)$"
+
+[tool.hatch.version.raw-options]
+version_scheme = "no-guess-dev"
+# This pyproject.toml lives two directories below the repo root (the
+# actual .git location); without an explicit root, setuptools-scm looks
+# for .git right next to pyproject.toml and fails outright rather than
+# walking up.
+root = "../.."
+# tag-pattern above only *extracts* a version from whatever tag `git
+# describe` returns -- it does not restrict which tags are candidates.
+# In this monorepo `git describe` finds the globally-nearest tag (e.g.
+# `dirsql-py-v0.3.110`), the regex correctly rejects it, and
+# setuptools-scm treats that as a hard error rather than falling back.
+# `--match` constrains `git describe` itself to this package's own
+# tag prefix.
+git_describe_command = ["git", "describe", "--dirty", "--tags", "--long", "--match", "dirsql-plugin-embeddings-v*"]
+
+[project.scripts]
+# The two command hooks the fragment invokes; installing the package puts them
+# on PATH so `dirsql` resolves them when it spawns the hooks.
+dirsql-embeddings-on-file = "dirsql_plugin_embeddings.on_file:main"
+dirsql-embeddings-pre-query = "dirsql_plugin_embeddings.pre_query:main"
+
+[project.entry-points.dirsql]
+# How the #529 launcher discovers the plugin: group `dirsql`, value = the
+# top-level module whose root ships `dirsql.toml`.
+embeddings = "dirsql_plugin_embeddings"
+
+[dependency-groups]
+dev = ["pytest>=8", "pytest-cov>=5", "pytest-describe>=2"]
+
+[tool.hatch.build]
+exclude = ["**/*_test.py"]
+
+[tool.hatch.build.targets.wheel]
+# The whole package ships, including the non-.py `dirsql.toml` fragment
+# (importlib.resources loads it, and it must be present in the built artifact).
+packages = ["src/dirsql_plugin_embeddings"]
+
+[tool.hatch.build.targets.wheel.force-include]
+"src/dirsql_plugin_embeddings/dirsql.toml" = "dirsql_plugin_embeddings/dirsql.toml"
+
+[tool.pytest.ini_options]
+# `src` layout: put it on the path so modules and their colocated tests import
+# the package normally (`from dirsql_plugin_embeddings import ...`).
+pythonpath = ["src"]

dirsql_plugin_embeddings-0.1.0/src/dirsql_plugin_embeddings/dirsql.toml

@@ -0,0 +1,18 @@
+# Fragment shipped by the dirsql-plugin-embeddings plugin.
+# Discovered via the package's [project.entry-points.dirsql] declaration and
+# injected by the launcher as an ordinary -c flag ("installed = active", #529).
+#
+# The table name here MUST match pre_query.TABLE_NAME (the SQL the pre-query
+# hook prints queries this table).
+[dirsql]
+pre-query = "dirsql-embeddings-pre-query {args}"
+hook-timeout = 300
+
+[[dirsql.extension]]
+path = "sqlite_vec"
+entrypoint = "sqlite3_vec_init"
+
+[[table]]
+ddl = "CREATE TABLE documents (path TEXT, text TEXT, embedding TEXT)"
+glob = "**/*.md"
+on-file = "dirsql-embeddings-on-file {path}"

dirsql_plugin_embeddings-0.1.0/src/dirsql_plugin_embeddings/embedder.py

@@ -0,0 +1,71 @@
+"""Embed text via an OpenAI-compatible ``/v1/embeddings`` endpoint.
+
+Configuration comes from three environment variables (base URL, model, API
+key). The HTTP call is behind an injected ``post`` seam so the unit tests drive
+it without a real network; ``_urllib_post`` is the production seam.
+
+Annotations are evaluated at runtime (no ``from __future__ import annotations``)
+so a mutated ``X | None`` union in a signature fails at import instead of being
+an inert string -- otherwise every annotation-union mutant would survive.
+"""
+
+import json
+import os
+import urllib.error
+import urllib.request
+from collections.abc import Mapping
+
+ENV_BASE_URL = "DIRSQL_EMBEDDINGS_BASE_URL"
+ENV_MODEL = "DIRSQL_EMBEDDINGS_MODEL"
+ENV_API_KEY = "DIRSQL_EMBEDDINGS_API_KEY"
+
+
+class EmbeddingError(RuntimeError):
+    """A configuration, transport, or response error while embedding."""
+
+
+def _urllib_post(url: str, data: bytes, headers: Mapping[str, str]):
+    request = urllib.request.Request(
+        url, data=data, headers=dict(headers), method="POST"
+    )
+    try:
+        with urllib.request.urlopen(request) as response:
+            return response.status, response.read()
+    except urllib.error.HTTPError as error:
+        return error.code, error.read()
+
+
+def _require(env: Mapping[str, str], name: str) -> str:
+    value = env.get(name, "")
+    if not value:
+        raise EmbeddingError(f"missing required environment variable {name}")
+    return value
+
+
+def embed(
+    text: str, env: Mapping[str, str] | None = None, post=_urllib_post
+) -> list[float]:
+    if env is None:
+        env = os.environ
+    base_url = _require(env, ENV_BASE_URL).rstrip("/")
+    model = _require(env, ENV_MODEL)
+    api_key = _require(env, ENV_API_KEY)
+
+    data = json.dumps({"model": model, "input": [text]}).encode("utf-8")
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {api_key}",
+    }
+    status, body = post(f"{base_url}/v1/embeddings", data=data, headers=headers)
+    if status != 200:
+        raise EmbeddingError(f"embeddings endpoint returned status {status}: {body!r}")
+
+    try:
+        entry = json.loads(body)["data"][0]
+    except (json.JSONDecodeError, KeyError, IndexError, TypeError) as exc:
+        raise EmbeddingError(f"malformed embeddings response: {body!r}") from exc
+
+    vector = entry.get("embedding")
+    if not isinstance(vector, list) or not vector:
+        raise EmbeddingError(f"embeddings response carried no vector: {body!r}")
+    return [float(component) for component in vector]

dirsql_plugin_embeddings-0.1.0/src/dirsql_plugin_embeddings/on_file.py

@@ -0,0 +1,33 @@
+"""``on-file`` console script: embed one matched file into a dirsql row.
+
+Reads the file at ``argv[1]``, embeds its text, and prints a one-line JSON row
+array (``path``, ``text``, ``embedding``) -- the embedding stored as JSON text,
+which ``sqlite-vec`` accepts directly.
+
+Annotations are evaluated at runtime (no ``from __future__ import annotations``)
+so a mutated ``X | None`` union in a signature fails at import rather than
+surviving as an inert string.
+"""
+
+import json
+import sys
+
+from .embedder import embed
+
+
+def _read_text(path: str) -> str:
+    with open(path, encoding="utf-8") as handle:
+        return handle.read()
+
+
+def build_rows(path: str, text: str, vector: list[float]) -> list[dict]:
+    return [{"path": path, "text": text, "embedding": json.dumps(vector)}]
+
+
+def main(argv: list[str] | None = None) -> int:
+    if argv is None:
+        argv = sys.argv
+    path = argv[1]
+    text = _read_text(path)
+    print(json.dumps(build_rows(path, text, embed(text))))
+    return 0

dirsql_plugin_embeddings-0.1.0/src/dirsql_plugin_embeddings/pre_query.py

@@ -0,0 +1,42 @@
+"""``pre-query`` console script: turn a ``{"q": ...}`` body into search SQL.
+
+Accepts both a verbatim server body (``{"q": ...}``) and the CLI ``query``
+subcommand's ``{"sql": <arg>}`` wrapper, embeds the question, and prints the
+nearest-neighbor SQL over the ``documents`` table (ordered by
+``vec_distance_cosine``). The hook owns SQL safety: the only interpolated value
+is a numeric vector this script produced.
+
+Annotations are evaluated at runtime (no ``from __future__ import annotations``)
+so a mutated ``X | None`` union in a signature fails at import rather than
+surviving as an inert string.
+"""
+
+import json
+import sys
+
+from .embedder import embed
+
+TABLE_NAME = "documents"
+RESULT_LIMIT = 3
+
+
+def question(raw_body: str) -> str:
+    body = json.loads(raw_body)
+    if "q" in body:
+        return body["q"]
+    return json.loads(body["sql"])["q"]
+
+
+def build_sql(vector: list[float]) -> str:
+    needle = json.dumps(vector)
+    return (
+        f"SELECT path, ROUND(vec_distance_cosine(embedding, '{needle}'), 3) "
+        f"AS distance FROM {TABLE_NAME} ORDER BY distance LIMIT {RESULT_LIMIT}"
+    )
+
+
+def main(argv: list[str] | None = None) -> int:
+    if argv is None:
+        argv = sys.argv
+    print(build_sql(embed(question(argv[1]))))
+    return 0

dirsql_plugin_embeddings-0.1.0/testing-conventions.toml

@@ -0,0 +1,7 @@
+# Dedicated testing-conventions config for the in-repo dirsql-plugin-embeddings
+# package. Mirrors internals/checks: python coverage floor 100 + branch, zero
+# exemptions, and deliberately no `[e2e] extra_scope` -- this package compiles
+# none of packages/rust/src, so a core change never stales its attestation.
+[python.coverage]
+fail_under = 100
+branch = true

dirsql_plugin_embeddings-0.1.0/tests/conftest.py

@@ -0,0 +1,62 @@
+"""Shared fixtures for the integration and e2e tiers.
+
+`stub_server` brings up a real, local, threaded OpenAI-compatible
+`/v1/embeddings` endpoint. Its embedding is a deterministic keyword-count
+vector, so nearest-neighbor search over the fixtures is reproducible without a
+real model or network.
+"""
+
+import json
+import threading
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+import pytest
+
+# One dimension per keyword; the embedding of a text is the per-keyword count.
+KEYWORDS = [
+    "pasta",
+    "cook",
+    "garlic",
+    "git",
+    "code",
+    "review",
+    "tomato",
+    "plant",
+    "seed",
+]
+
+
+def keyword_vector(text):
+    lowered = text.lower()
+    return [float(lowered.count(keyword)) for keyword in KEYWORDS]
+
+
+class _Handler(BaseHTTPRequestHandler):
+    def do_POST(self):
+        length = int(self.headers["Content-Length"])
+        payload = json.loads(self.rfile.read(length))
+        text = payload["input"][0]
+        body = json.dumps(
+            {"data": [{"index": 0, "embedding": keyword_vector(text)}]}
+        ).encode("utf-8")
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def log_message(self, *args):  # silence the default stderr access log
+        pass
+
+
+@pytest.fixture
+def stub_server():
+    server = HTTPServer(("127.0.0.1", 0), _Handler)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    host, port = server.server_address
+    try:
+        yield f"http://{host}:{port}"
+    finally:
+        server.shutdown()
+        thread.join()