ph-code-graph 0.2.0__tar.gz → 0.3.0__tar.gz

{ph_code_graph-0.2.0 → ph_code_graph-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: ph-code-graph
-Version: 0.2.0
+Version: 0.3.0
 Summary: pH plugin: `code_index` and `code_graph`, a Python-native tree-sitter code graph with line-accurate answers.
 Project-URL: Homepage, https://github.com/chastabor/pH
 Project-URL: Repository, https://github.com/chastabor/pH
@@ -21,7 +21,7 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.12
-Requires-Dist: ph-core==0.2.0
+Requires-Dist: ph-core==0.3.0
 Requires-Dist: tree-sitter-language-pack>=1.16
 Requires-Dist: tree-sitter>=0.25
 Description-Content-Type: text/markdown

{ph_code_graph-0.2.0 → ph_code_graph-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "ph-code-graph"
-version = "0.2.0"
+version = "0.3.0"
 description = "pH plugin: `code_index` and `code_graph`, a Python-native tree-sitter code graph with line-accurate answers."
 readme = "README.md"
 requires-python = ">=3.12"
@@ -29,7 +29,7 @@ classifiers = [
 # in (371 available, the rest fetched on demand — which this row never does
 # implicitly; see `_extract.indexable`). No Rust toolchain, no Node, no
 # submodule: the graph itself is stdlib `sqlite3`.
-dependencies = ["ph-core==0.2.0", "tree-sitter>=0.25", "tree-sitter-language-pack>=1.16"]
+dependencies = ["ph-core==0.3.0", "tree-sitter>=0.25", "tree-sitter-language-pack>=1.16"]
 
 [project.urls]
 Homepage = "https://github.com/chastabor/pH"

{ph_code_graph-0.2.0 → ph_code_graph-0.3.0}/src/ph_code_graph/__init__.py

@@ -43,6 +43,7 @@ from __future__ import annotations
 
 import hashlib
 import logging
+from collections.abc import Mapping
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Literal
@@ -59,6 +60,7 @@ from ph.seams._registry import contribute_item
 from ph.seams.changes import tree_state
 from ph.seams.commands import CommandContext, CommandDefinition
 from ph.seams.diagnostics import Diagnostic, contribute
+from ph.seams.fs import FsService
 from ph.seams.skills import discover_skills
 from ph.text import count_of
 from ph.tools.definition import ToolModel, ToolOutput, ToolRunContext, define_tool, text_content
@@ -535,6 +537,16 @@ async def apply(ctx: Context, config: Config) -> None:
     def store(run: ToolRunContext) -> CodeGraphStore:
         return seam.store_for(ctx.require(FS).root_for(run.agent))
 
+    def _forget_missing(
+        fs: FsService,
+        book: CodeGraphStore,
+        known: Mapping[str, tuple[str, str]],
+        run: ToolRunContext,
+    ) -> int:
+        """Drop every indexed path that is no longer on disk. One thread hop."""
+        gone = [path for path in known if not fs.resolve(path, agent=run.agent).exists()]
+        return book.forget(gone) if gone else 0
+
     async def index_tool(args: IndexArgs, run: ToolRunContext) -> dict[str, Any]:
         fs = ctx.require(FS)
         book = store(run)
@@ -616,10 +628,24 @@ async def apply(ctx: Context, config: Config) -> None:
                 slice_ = await fs.read(
                     path,
                     limit=None,
+                    # `skip_reason` above is this walk's bound, and it is a skip
+                    # rather than a raise on purpose: an indexing pass must not
+                    # fail because it found a minified bundle. `read`'s own
+                    # default would be a second, larger bound that raises.
+                    max_bytes=None,
                     scope=run.scope,
                     agent=run.agent,
                     session=run.session,
                 )
+                # **No prose test here, deliberately** (X6 review). The text
+                # index refuses documents that hold no prose; source code is not
+                # prose, and the same predicate refused a module whose only
+                # statement is a long `__all__`, and a `TABLE = {...}` literal
+                # beside `def lookup()` — so `lookup` would have vanished from
+                # this index with nothing but a skip line to say so. Telling a
+                # minified bundle from a legitimate data module by shape is not
+                # reliable for code, and a false positive here loses symbols
+                # silently. `skip_reason`'s size bound stays this walk's guard.
                 # The content hash stays the authority on *whether* a file
                 # changed — the filter above only decides whether to open it, so
                 # the "content, not clock" guarantee is untouched.
@@ -642,6 +668,25 @@ async def apply(ctx: Context, config: Config) -> None:
                     book.put, path, digest, extraction, state.id_for(path)
                 )
                 indexed += 1
+            if not args.forget:
+                # **Gone from disk is gone from the index** (X3). A deleted file
+                # kept its symbols until somebody ran `forget` by hand, so
+                # `code_graph` went on naming definitions at paths that are not
+                # there — and a rename showed the symbol twice, once under each
+                # name, with nothing saying which was real. A pointer into a file
+                # that does not exist is the one answer this tool must not give,
+                # because the model's next move is to open it.
+                #
+                # **Asked of the filesystem, not of the walk.** "Absent from
+                # `paths`" looks like the same question and is not: `collect`
+                # stops at `max_files` and reports nothing, and the policy screen
+                # prunes whole directories — so on a tree past the cap, or with a
+                # `permissions-fs` rule hiding a subtree, the walk is a *subset*
+                # of the scope and differencing against it would forget
+                # everything it did not reach. It also settles the narrowed walk
+                # for free, where the question is not "did we cover the scope"
+                # but "is this file still there".
+                removed = await anyio.to_thread.run_sync(_forget_missing, fs, book, known, run)
             # Stored last, and only after the loop: a token recorded before the
             # writes would, on a crash between the two, vouch for files this run
             # never actually indexed.

{ph_code_graph-0.2.0 → ph_code_graph-0.3.0}/src/ph_code_graph/_store.py

@@ -213,6 +213,22 @@ class Hit:
         }
 
 
+def _fts_values(rowid: int, name: str, doc: str | None, path: str) -> tuple[int, str, str, str]:
+    """One `symbols_fts` row, for the insert **and** the delete that undoes it.
+
+    FTS5's `'delete'` command re-derives the postings to remove from the values
+    it is given, so a delete whose tuple differs from the insert's by so much as
+    one NULL removes nothing — and SQLite reports no error for it. `_forget`
+    carried a copy of this expression with a comment saying it mirrored the
+    insert, which is a reader being asked to check two expressions in two
+    methods rather than a compiler being asked to.
+
+    `doc or ""` is the whole substance: `symbols.doc` is nullable and the FTS
+    column is not.
+    """
+    return (rowid, name, doc or "", path)
+
+
 @dataclass(slots=True)
 class CodeGraphStore:
     """The index. **Blocking**; the seam calls it in a worker thread."""
@@ -361,17 +377,33 @@ class CodeGraphStore:
     def _forget(self, connection: sqlite3.Connection, paths: Sequence[str]) -> int:
         gone = 0
         for path in paths:
-            # The FTS rows first, by the id they were inserted under: a
+            # The FTS rows first, **with the values they were inserted under**: a
             # contentless table cannot be asked which rows belong to a path, so
-            # the ids come from `symbols` while it still has them.
-            ids = [
-                row["id"]
-                for row in connection.execute("SELECT id FROM symbols WHERE path = ?", (path,))
+            # they come from `symbols` while it still has them (X1).
+            #
+            # The columns and not just the rowid, because that is what FTS5's
+            # `'delete'` command needs: it re-derives the postings to remove from
+            # the values it is given, and three empty strings therefore remove
+            # *nothing*. SQLite reports no error for it. Every posting of every
+            # re-indexed symbol stayed, and `symbols.id` has no `AUTOINCREMENT` —
+            # so the next insert took the dead row's id and the stale posting
+            # started pointing at a different symbol. `search` joins the rowid
+            # back to `symbols`, so the old name returned the new symbol: worse
+            # than a miss, because it reads like an answer.
+            #
+            # Both tuples come from `_fts_values`, so the agreement the paragraph
+            # above depends on is structural rather than a comment asking a
+            # reader to check two expressions in two methods.
+            stale = [
+                _fts_values(row["id"], row["name"], row["doc"], row["path"])
+                for row in connection.execute(
+                    "SELECT id, name, doc, path FROM symbols WHERE path = ?", (path,)
+                )
             ]
             connection.executemany(
                 "INSERT INTO symbols_fts(symbols_fts, rowid, name, doc, path) "
-                "VALUES ('delete', ?, '', '', '')",
-                [(one,) for one in ids],
+                "VALUES ('delete', ?, ?, ?, ?)",
+                stale,
             )
             connection.execute("DELETE FROM symbols WHERE path = ?", (path,))
             connection.execute("DELETE FROM refs WHERE path = ?", (path,))
@@ -420,7 +452,7 @@ class CodeGraphStore:
                 by_line[definition.start_line] = identifier
                 connection.execute(
                     "INSERT INTO symbols_fts(rowid, name, doc, path) VALUES (?, ?, ?, ?)",
-                    (identifier, definition.name, definition.doc or "", path),
+                    _fts_values(identifier, definition.name, definition.doc, path),
                 )
             # One `{line: tightest definition}` table for the whole file, rather
             # than a scan of every definition per reference: that was

{ph_code_graph-0.2.0 → ph_code_graph-0.3.0}/tests/test_code_graph.py

@@ -539,6 +539,48 @@ async def test_an_edit_is_picked_up_and_replaces_the_old_rows(
     assert here.value["symbols"], "the new symbol was not indexed"
 
 
+async def test_a_renamed_symbol_is_not_found_under_its_old_name(
+    mount: MountProfile, tmp_path: Path
+) -> None:
+    """The full-text index has to forget too, and it was not (X1).
+
+    `symbols_fts` is contentless, and a contentless FTS5 table cannot be asked
+    which postings belong to a row — the `'delete'` command needs the column
+    values the row was *inserted* with, and it was being handed three empty
+    strings. SQLite takes that quietly and removes nothing, so every posting for
+    every symbol ever re-indexed stayed in the index.
+
+    Stale postings would be merely wasteful if the rowids were retired; they are
+    not. `symbols.id` has no `AUTOINCREMENT`, so the next insert reuses the id
+    the deleted symbol had, and the old posting now points at a *different*
+    symbol. `search` joins the FTS rowid back to `symbols`, so asking for the old
+    name returns the new symbol — which is worse than a miss, because it reads
+    like an answer.
+
+    `define` was already covered by the sibling test above and passes either way:
+    it queries `symbols` directly and never touches the FTS table. `search` is
+    the mode that goes through it, which is why the defect survived.
+    """
+    # **One file, and that is load-bearing.** `symbols.id` is reused only when
+    # the deleted row held the highest one; with a second file indexed after it,
+    # the stale posting points at an id nothing has taken back and the join in
+    # `search` drops it, hiding the defect behind a miss.
+    _tree(tmp_path)
+    ctx = await _indexed(mount, tmp_path)
+    agent = _agent(ctx)
+    await run_tool(ctx, "code_index", {"paths": ["pkg/helpers.py"]}, agent=agent)
+
+    (tmp_path / "pkg" / "helpers.py").write_text(
+        "def gamma(value):\n    return value\n", encoding="utf-8"
+    )
+    await run_tool(ctx, "code_index", {"paths": ["pkg/helpers.py"]}, agent=agent)
+
+    stale = await run_tool(ctx, "code_graph", {"mode": "search", "query": "shared"}, agent=agent)
+    assert stale.value["symbols"] == [], "the old name still has postings in the index"
+    found = await run_tool(ctx, "code_graph", {"mode": "search", "query": "gamma"}, agent=agent)
+    assert [one["name"] for one in found.value["symbols"]] == ["gamma"]
+
+
 async def test_forget_removes_a_path_from_the_index(mount: MountProfile, tmp_path: Path) -> None:
     _tree(tmp_path)
     ctx = await _indexed(mount, tmp_path)
@@ -1191,3 +1233,116 @@ async def test_a_tree_with_no_version_control_still_indexes(
     assert first.value["indexed"] == 2, "the fixture's two modules"
     # Still reported unchanged — by the content hash, which never went away.
     assert again.value["indexed"] == 0 and again.value["unchanged"] == 2
+
+
+async def test_a_deleted_file_leaves_the_index_on_the_next_sweep(
+    mount: MountProfile, tmp_path: Path
+) -> None:
+    """X3 — the walk never diffed itself against what the index already held.
+
+    A file removed from the tree kept its symbols until somebody ran `forget` by
+    hand, so `code_graph` went on naming definitions at paths that are not there
+    — and a *rename* showed the symbol twice, once under each name, with nothing
+    saying which was real. A pointer into a file that does not exist is the one
+    answer this tool must not give, because the model's next move is to open it.
+
+    Only a full sweep does this, which is the second half: narrowing the walk
+    must not empty the index, or `code_index paths=["src/x.py"]` would delete
+    everything else.
+    """
+    _tree(tmp_path)
+    ctx = await _indexed(mount, tmp_path)
+    agent = _agent(ctx)
+    await run_tool(ctx, "code_index", {"paths": ["."]}, agent=agent)
+
+    (tmp_path / "pkg" / "helpers.py").unlink()
+    swept = await run_tool(ctx, "code_index", {"paths": ["."]}, agent=agent)
+
+    assert not swept.is_error, text_of(swept.content)
+    assert swept.value["removed"] == 1, "the deleted file kept its symbols"
+    found = await run_tool(ctx, "code_graph", {"mode": "define", "query": "shared"}, agent=agent)
+    assert found.value["symbols"] == [], "a definition still points at a file that is gone"
+
+
+async def test_a_walk_that_did_not_reach_a_file_does_not_forget_it(
+    mount: MountProfile, tmp_path: Path
+) -> None:
+    """X3's guard rail, and the reason the sweep asks the filesystem.
+
+    The first cut of this fix differenced the index against the *walk* and
+    decided it was a full sweep by pattern-matching the arguments
+    (`paths == ["."]`). Two things make that wrong, and both empty the index
+    rather than trimming it: `fs.collect` stops at `max_files` and reports
+    nothing, so on a tree past the cap the walk is a prefix of the scope; and
+    the `permissions-fs` screen prunes whole directories, so a rule hiding a
+    subtree makes those files "absent". Asking `exists()` settles all three —
+    truncated, pruned and narrowed — because "is this file still there" is the
+    question, and it never depended on the walk at all.
+    """
+    _tree(tmp_path)
+    ctx = await _indexed(mount, tmp_path)
+    agent = _agent(ctx)
+    await run_tool(ctx, "code_index", {"paths": ["."]}, agent=agent)
+
+    narrowed = await run_tool(ctx, "code_index", {"paths": ["pkg/helpers.py"]}, agent=agent)
+
+    assert narrowed.value["removed"] == 0
+    still = await run_tool(ctx, "code_graph", {"mode": "define", "query": "shared"}, agent=agent)
+    assert still.value["symbols"], "a narrowed walk emptied the index"
+
+    # And a walk truncated by `max_files`, which *looks* like a full sweep: the
+    # arguments are the row's own defaults and `paths` is still only a subset.
+    capped = await _indexed(mount, tmp_path / "capped", max_files=1)
+    narrow_agent = _agent(capped)
+    await run_tool(capped, "code_index", {"paths": ["."]}, agent=narrow_agent)
+    swept = await run_tool(capped, "code_index", {"paths": ["."]}, agent=narrow_agent)
+
+    assert swept.value["removed"] == 0, "a truncated walk forgot the files it never reached"
+
+
+async def test_a_forgotten_symbols_docstring_stops_being_searchable(
+    mount: MountProfile, tmp_path: Path
+) -> None:
+    """N5 — the FTS delete has to carry the values the insert used.
+
+    FTS5's `'delete'` re-derives the postings to remove from the values it is
+    handed, so a delete that passes different values removes only the postings
+    those values imply. Measured against SQLite 3.45: deleting with `""` for a
+    symbol inserted with real docstring text removes the **name** posting and
+    leaves every **doc** posting behind.
+
+    A stale posting is worse than a miss: `symbols.id` has no `AUTOINCREMENT`,
+    so a later insert takes the dead row's id and the old docstring starts
+    answering with a different symbol — which is what this stages, because
+    `search` joins the rowid back to `symbols` and a posting whose symbol is
+    simply gone is invisible.
+
+    Driven through `code_index`/`code_graph` rather than against `_fts_values`,
+    which is how this first shipped: the helper was written, exported to a unit
+    test, and never called, so the test passed while both live expressions
+    stayed inline.
+
+    Sabotage: pass `""` as the doc at either call site and `zarquon` answers
+    with the symbol that took the id.
+    """
+    _tree(tmp_path)
+    (tmp_path / "pkg" / "old.py").write_text(
+        'def retired():\n    """A zarquon of the third kind."""\n    return 1\n', encoding="utf-8"
+    )
+    ctx = await _indexed(mount, tmp_path)
+    agent = _agent(ctx)
+    await run_tool(ctx, "code_index", {"paths": ["pkg"]}, agent=agent)
+    seen = await run_tool(ctx, "code_graph", {"mode": "search", "query": "zarquon"}, agent=agent)
+    assert [one["name"] for one in seen.value["symbols"]] == ["retired"], "never indexed"
+
+    # Forget it, then index a *new* symbol that recycles the dead row's id.
+    (tmp_path / "pkg" / "old.py").unlink()
+    await run_tool(ctx, "code_index", {"paths": ["pkg/old.py"], "forget": True}, agent=agent)
+    (tmp_path / "pkg" / "new.py").write_text("def arrived():\n    return 2\n", encoding="utf-8")
+    await run_tool(ctx, "code_index", {"paths": ["pkg"]}, agent=agent)
+
+    gone = await run_tool(ctx, "code_graph", {"mode": "search", "query": "zarquon"}, agent=agent)
+
+    assert gone.value["symbols"] == [], (
+        "a docstring posting outlived its symbol and answered for the one that took its id"
+    )