dirsql-plugin-embeddings 0.1.14__tar.gz → 0.1.16__tar.gz

{dirsql_plugin_embeddings-0.1.14 → dirsql_plugin_embeddings-0.1.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: dirsql-plugin-embeddings
-Version: 0.1.14
+Version: 0.1.16
 Summary: First-party dirsql plugin: semantic search over files.
 Requires-Python: >=3.10
 Requires-Dist: cachetta>=0.7
@@ -26,10 +26,15 @@ uvx --with dirsql-plugin-embeddings dirsql "
   SELECT path
   FROM (SELECT path, embed(content ->> 'abstract') AS emb
         FROM './arxiv-firehose/data/**/metadata.json')
+  WHERE emb IS NOT NULL
   ORDER BY vec_distance_cosine(emb, embed('local private models'))
   LIMIT 10"
 ```
 
+`WHERE emb IS NOT NULL` is not optional bookkeeping: a file that is unreadable
+or not valid UTF-8 has NULL content, so its distance is NULL, and SQLite sorts
+NULLs *first* ascending — without the guard those files take the top slots.
+
 For the common case — one glob, one question, top-k paths — the package is
 also its own command, generating and running exactly that SQL:
 

{dirsql_plugin_embeddings-0.1.14 → dirsql_plugin_embeddings-0.1.16}/README.md

@@ -13,10 +13,15 @@ uvx --with dirsql-plugin-embeddings dirsql "
   SELECT path
   FROM (SELECT path, embed(content ->> 'abstract') AS emb
         FROM './arxiv-firehose/data/**/metadata.json')
+  WHERE emb IS NOT NULL
   ORDER BY vec_distance_cosine(emb, embed('local private models'))
   LIMIT 10"
 ```
 
+`WHERE emb IS NOT NULL` is not optional bookkeeping: a file that is unreadable
+or not valid UTF-8 has NULL content, so its distance is NULL, and SQLite sorts
+NULLs *first* ascending — without the guard those files take the top slots.
+
 For the common case — one glob, one question, top-k paths — the package is
 also its own command, generating and running exactly that SQL:
 

dirsql_plugin_embeddings-0.1.16/changelog.d/2026-08-12-empty-search-reports-why.md

@@ -0,0 +1,3 @@
+**Fixed**
+
+- **A search that ranks nothing now says why and exits nonzero, instead of printing nothing and exiting 0.** Silent empty output was indistinguishable from "searched and found nothing relevant" — the reported case was running `dirsql-plugin-embeddings 'docs/**/*.md' <query>` from inside `docs/`, where the glob matches nothing. The message names the glob and the directory it searched from (`no files matched 'docs/**/*.md' (searched from /current/dir)`), and distinguishes that from a corpus whose files all matched but had no embeddable text, so it never claims the wrong reason. (#816)

dirsql_plugin_embeddings-0.1.16/changelog.d/2026-08-12-null-embedding-guard.md

@@ -0,0 +1,3 @@
+**Fixed**
+
+- **Files that cannot be embedded no longer take the top of the ranking (and no longer break the search).** A matched file that is unreadable, not valid UTF-8, or deleted mid-scan has NULL `content`, so `embed(content)` is NULL — and SQLite sorts NULLs *first* ascending, so those files occupied the top-k slots before anything else went wrong. The generated search SQL now carries `WHERE emb IS NOT NULL`, which fixes the ranking and the failure it led to (a `TypeError` formatting a NULL distance, or, depending on the sqlite-vec build, `Error reading 1st vector: ... found NULL` from SQLite itself). The documented hand-written idiom carries the same guard. (#817)

dirsql_plugin_embeddings-0.1.16/e2e-attestations/claude-816-empty-corpus-error.json

@@ -0,0 +1,7 @@
+{
+  "command": "uv run --with-editable ../../packages/python python -m pytest tests/e2e/ -x -q",
+  "ran_at": 1786553959,
+  "exit_code": 0,
+  "commit": "c349f1ed7b83b0817d68019550264edb0a255ddd",
+  "branch": "claude/816-empty-corpus-error"
+}

dirsql_plugin_embeddings-0.1.16/e2e-attestations/claude-817-null-embedding-guard.json

@@ -0,0 +1,7 @@
+{
+  "command": "uv run --with-editable ../../packages/python python -m pytest tests/e2e/ -x -q",
+  "ran_at": 1786558054,
+  "exit_code": 0,
+  "commit": "f1af2bf7bf4c21dd62350b48b4d441356047ef59",
+  "branch": "claude/817-null-embedding-guard"
+}

{dirsql_plugin_embeddings-0.1.14 → dirsql_plugin_embeddings-0.1.16}/src/dirsql_plugin_embeddings/cli/search.py

@@ -1,6 +1,6 @@
 import click
 
-from ..search.run import run_search
+from ..search.run import NothingToRank, run_search
 
 
 @click.command(hidden=True)
@@ -22,5 +22,10 @@ from ..search.run import run_search
 )
 def search(glob, query, limit, model):
     """Rank files matching GLOB by semantic similarity to QUERY."""
-    for line in run_search(glob, query, limit, model):
+    try:
+        lines = run_search(glob, query, limit, model)
+    except NothingToRank as error:
+        click.echo(f"dirsql-plugin-embeddings: {error}", err=True)
+        raise SystemExit(1) from None
+    for line in lines:
         click.echo(line)

dirsql_plugin_embeddings-0.1.16/src/dirsql_plugin_embeddings/search/run.py

@@ -0,0 +1,45 @@
+import asyncio
+import os
+from importlib import resources
+
+import dirsql
+
+from .output import format_rows
+from .sql import build_search_sql, count_corpus_sql
+
+
+class NothingToRank(Exception):
+    """The search produced no rows, and why -- so the CLI can say which."""
+
+
+def config_fragment():
+    return str(resources.files("dirsql_plugin_embeddings").joinpath("dirsql.toml"))
+
+
+def no_rows_message(glob, matched, root):
+    if not matched:
+        return f"no files matched {glob!r} (searched from {root})"
+    return (
+        f"{glob!r} matched {matched} file(s), but none had text content to"
+        f" embed -- unreadable or not valid UTF-8 (searched from {root})"
+    )
+
+
+async def _search(glob, query, limit, model):
+    db = dirsql.DirSQL(config=config_fragment())
+    rows = await db.query(build_search_sql(glob, query, limit, model))
+    if rows:
+        return rows, None
+    # Only now, on the error path, is the second scan worth its cost: it is
+    # what tells "no files matched" apart from "matched, none embeddable".
+    # One row, always: `SELECT COUNT(*)` cannot return anything else, and
+    # unpacking says so rather than trusting an index.
+    (counted,) = await db.query(count_corpus_sql(glob))
+    return rows, counted["n"]
+
+
+def run_search(glob, query, limit, model=None):
+    rows, matched = asyncio.run(_search(glob, query, limit, model))
+    if not rows:
+        raise NothingToRank(no_rows_message(glob, matched, os.getcwd()))
+    return format_rows(rows)

{dirsql_plugin_embeddings-0.1.14 → dirsql_plugin_embeddings-0.1.16}/src/dirsql_plugin_embeddings/search/sql.py

@@ -25,8 +25,17 @@ def embed_call(argument, model):
 def build_search_sql(glob, query, limit, model=None):
     outer = embed_call(quote(query), model)
     inner = embed_call("content", model)
+    # A matched file that is unreadable, not valid UTF-8, or deleted mid-scan
+    # has NULL content, so its embedding is NULL too. Dropping those rows is
+    # what keeps the ranking honest: a NULL distance sorts FIRST ascending in
+    # SQLite, so unrankable files would otherwise take the top-k slots.
     return (
         f"SELECT path, vec_distance_cosine(emb, {outer}) AS distance"
         f" FROM (SELECT path, {inner} AS emb FROM {quote(normalize_glob(glob))})"
+        f" WHERE emb IS NOT NULL"
         f" ORDER BY distance LIMIT {int(limit):d}"
     )
+
+
+def count_corpus_sql(glob):
+    return f"SELECT COUNT(*) AS n FROM {quote(normalize_glob(glob))}"

dirsql_plugin_embeddings-0.1.14/src/dirsql_plugin_embeddings/search/run.py

@@ -1,20 +0,0 @@
-import asyncio
-from importlib import resources
-
-import dirsql
-
-from .output import format_rows
-from .sql import build_search_sql
-
-
-def config_fragment():
-    return str(resources.files("dirsql_plugin_embeddings").joinpath("dirsql.toml"))
-
-
-async def _query(sql):
-    return await dirsql.DirSQL(config=config_fragment()).query(sql)
-
-
-def run_search(glob, query, limit, model=None):
-    rows = asyncio.run(_query(build_search_sql(glob, query, limit, model)))
-    return format_rows(rows)