PyPI - sql-code-graph - Versions diffs - 1.2.2__py3-none-any.whl → 1.4.0__py3-none-any.whl - Mend

sql-code-graph 1.2.2py3-none-any.whl → 1.4.0py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (33) hide show

{sql_code_graph-1.2.2.dist-info → sql_code_graph-1.4.0.dist-info}/METADATA +2 -4
{sql_code_graph-1.2.2.dist-info → sql_code_graph-1.4.0.dist-info}/RECORD +31 -30
sqlcg/__init__.py +1 -1
sqlcg/cli/commands/analyze.py +138 -127
sqlcg/cli/commands/db.py +49 -51
sqlcg/cli/commands/find.py +5 -9
sqlcg/cli/commands/gain.py +14 -16
sqlcg/cli/commands/git.py +11 -4
sqlcg/cli/commands/index.py +173 -21
sqlcg/cli/commands/mcp.py +70 -3
sqlcg/cli/commands/reindex.py +147 -77
sqlcg/cli/commands/uninstall.py +9 -20
sqlcg/core/__init__.py +1 -3
sqlcg/core/config.py +25 -81
sqlcg/core/duckdb_backend.py +764 -0
sqlcg/core/freshness.py +1 -1
sqlcg/core/graph_db.py +20 -4
sqlcg/core/queries.py +26 -7
sqlcg/core/queries.sql +249 -0
sqlcg/core/schema.py +1 -1
sqlcg/indexer/indexer.py +27 -36
sqlcg/metrics/store.py +49 -1
sqlcg/server/control.py +1 -1
sqlcg/server/noise_filter.py +1 -1
sqlcg/server/read_client.py +2 -2
sqlcg/server/server.py +184 -86
sqlcg/server/skill.py +2 -2
sqlcg/server/tools.py +119 -41
sqlcg/server/writer.py +459 -0
sqlcg/core/kuzu_backend.py +0 -445
sqlcg/core/neo4j_backend.py +0 -233
{sql_code_graph-1.2.2.dist-info → sql_code_graph-1.4.0.dist-info}/WHEEL +0 -0
{sql_code_graph-1.2.2.dist-info → sql_code_graph-1.4.0.dist-info}/entry_points.txt +0 -0

sqlcg/server/server.py CHANGED Viewed

@@ -79,37 +79,51 @@ async def _control_socket_task(
     stop_event: "anyio.Event",
     backend_lock: "anyio.Lock",
     start_time: float,
+    writer_queue: "WriterQueue",
 ) -> None:
     """Accept control connections on ``<db>.sock`` and dispatch ops.
     Supported ops:
-    - ``{"op": "status"}`` → running state, pid, db_path, freshness, uptime.
-      Unframed (legacy single-recv protocol).
+    - ``{"op": "status"}`` → running state, pid, db_path, freshness, uptime,
+      writer_queue block.  **Length-prefixed framing** (v1.3.0, B3): the
+      response uses ``<decimal-byte-length>\\n<json-body>`` so large queue
+      payloads are read in full by the recv-exactly client.
     - ``{"op": "stop"}`` → sends ``{"ok": true}`` then signals stop via
-      *stop_event*.  Unframed.
-    - ``{"op": "reindex", "root", "from", "to", "dialect"}`` → runs
-      ``Indexer.resync_changed`` off the event-loop thread via
-      ``anyio.to_thread.run_sync``, serialised behind *backend_lock* (R1, R2).
-      Unframed.
+      *stop_event*.  Unframed (mcp_stop uses s.recv(128) — do NOT change).
+    - ``{"op": "index", "root", "dialect", "wait"}`` → enqueues a full index
+      onto *writer_queue* (rule 1 — supersedes all pending).  Supports
+      ``wait=true`` (stream progress frames + terminal ``done:true``) and
+      ``wait=false`` (immediate ``{ok, queued, position}``).
+    - ``{"op": "reindex", "root", "from", "to", "dialect", "wait"}`` →
+      enqueues an incremental resync (coalescing rules 2–3). ``from`` may be
+      ``null``/omitted to resolve at drain start (W3).  Same ``wait`` semantics
+      as ``index``.  The handler enqueues only — it never touches the backend
+      (B1 invariant: only the drain task resolves a backend, under backend_lock).
     - ``{"op": "query", "cypher": ..., "params": ...}`` → executes a
-      read-only Cypher query on the single backend connection, serialised
-      behind *backend_lock*.  **Length-prefixed framing** (v1.2.0):
+      read-only SQL query on the single backend connection, serialised
+      behind *backend_lock*.  (The ``cypher`` field name is a legacy wire-key
+      retained for protocol compatibility; the value is SQL.)
+      **Length-prefixed framing** (v1.2.0):
       ``<decimal-byte-length>\\n<json-body>`` on both request and response.
-    Framing protocol (v1.2.0, ``query`` op only):
-      Request: ``b"<len>\\n" + json_body`` — server detects by sniffing the
-      first line; a bare decimal integer → framed.  Unframed requests always
-      start with ``{`` (never a digit), so the sniff is unambiguous.
-      Response: same ``<len>\\n<body>`` format for framed requests; unframed
-      response for unframed requests.  Old clients that use the unframed
-      ``s.recv(65536)`` + ``json.loads`` pattern will get a loud
-      ``json.JSONDecodeError`` if they accidentally receive a framed response —
-      NOT silent truncation.  Only the new ``read_client`` sends framed
-      requests, so this does not affect existing callers.
+    Framing protocol:
+      Requests: a bare decimal integer on the first line → framed.  Unframed
+      JSON always starts with ``{``, so the sniff is unambiguous.
+      Responses: framed (``<len>\\n<body>``) for ``query`` and ``status``;
+      unframed for ``stop``/``reindex``/``index`` (unless ``wait=true`` which
+      uses the multi-frame streaming protocol).
+    Multi-frame streaming protocol (``index``/``reindex`` with ``wait=true``):
+      The server sends a sequence of length-prefixed frames on the same
+      connection.  Progress frames carry ``{done: false, files_done, files_total}``.
+      The terminal frame carries ``{ok: true, done: true, summary: {...}}`` on
+      success or ``{ok: false, done: true, error: ...}`` on failure (W7).
+      The client reads frames in a loop and stops when it sees ``done == true``
+      — it does NOT rely on EOF as the terminator.
     R2 (single connection): all backend operations go through ``backend_lock``
-    so concurrent calls never touch the single Kuzu connection simultaneously.
+    so concurrent calls never touch the single DuckDB connection simultaneously.
     R8 teardown ordering: the caller must cancel this task BEFORE calling
     ``shutdown_backend()``.  This is guaranteed by the ``anyio.CancelScope``
@@ -126,18 +140,19 @@ async def _control_socket_task(
     from anyio.streams.buffered import BufferedByteReceiveStream
     from sqlcg.core.config import get_db_path as _get_db_path
+    from sqlcg.server.writer import WriterRequest
-    # Read-only keyword allow-list for the ``query`` op.  Only these leading
-    # keywords are permitted — anything that starts with a write keyword is
-    # rejected before execution.  This is a guard against accidental mutation,
+    # Read-only keyword allow-list for the ``query`` op.  Only SELECT and WITH
+    # (CTE preamble) are permitted — anything that starts with a write keyword
+    # is rejected before execution.  This is a guard against accidental mutation,
     # not a security boundary (the socket is already 0o600 / owner-only).
-    _QUERY_ALLOWED_KEYWORDS = frozenset({"MATCH", "RETURN", "WITH", "CALL", "UNWIND", "OPTIONAL"})
+    _QUERY_ALLOWED_KEYWORDS = frozenset({"SELECT", "WITH", "VALUES", "TABLE"})
-    def _is_read_only_cypher(cypher: str) -> bool:
+    def _is_read_only_sql(sql: str) -> bool:
         """Return True iff the leading keyword is in the read-only allow-list."""
         import re
-        m = re.match(r"\s*(?:--[^\n]*)?\s*(\w+)", cypher, re.IGNORECASE)
+        m = re.match(r"\s*(?:--[^\n]*)?\s*(\w+)", sql, re.IGNORECASE)
         if not m:
             return False
         return m.group(1).upper() in _QUERY_ALLOWED_KEYWORDS
@@ -192,7 +207,7 @@ async def _control_socket_task(
                         if indexed_sha is not None:
                             try:
                                 rows = db.run_read(
-                                    "MATCH (r:Repo) RETURN r.path AS path LIMIT 1",
+                                    'SELECT path FROM "Repo" LIMIT 1',
                                     {},
                                 )
                                 if rows:
@@ -211,7 +226,13 @@ async def _control_socket_task(
                         "stale_by_commits": stale,
                         "connected_clients": 1,  # stdio transport = 1 by design
                         "uptime": time.time() - start_time,
+                        "writer_queue": writer_queue.coalesce_view(),
                     }
+                    # status response is framed (B3, v1.3.0) — same framing as query
+                    # so recv-exactly clients read it in full regardless of payload size.
+                    resp_bytes = json.dumps(resp).encode()
+                    await stream.send(f"{len(resp_bytes)}\n".encode() + resp_bytes)
+                    return
                 elif op == "stop":
                     resp = {"ok": True}
@@ -222,42 +243,99 @@ async def _control_socket_task(
                     stop_event.set()
                     return
+                elif op == "index":
+                    # Step 3.1 — enqueue a full index; never touches the backend here (B1).
+                    root = req.get("root")
+                    dialect = req.get("dialect")
+                    wait = req.get("wait", False)
+                    requested_by = req.get("requested_by", "cli")
+                    if not root:
+                        resp = {"error": "index op requires root"}
+                        await stream.send(json.dumps(resp).encode() + b"\n")
+                        return
+                    writer_req = WriterRequest(
+                        op="index",
+                        root=root,
+                        dialect=dialect,
+                        from_sha=None,
+                        to_sha=None,
+                        requested_by=requested_by,
+                    )
+                    if wait:
+                        # Attach-and-wait: register a memory channel then stream frames.
+                        send_ch, recv_ch = anyio.create_memory_object_stream(max_buffer_size=64)
+                        writer_req._waiters.append(send_ch)
+                        position = await writer_queue.enqueue(writer_req)
+                        # Send the queued acknowledgement frame first.
+                        queued_frame = json.dumps(
+                            {"ok": True, "done": False, "queued": True, "position": position}
+                        ).encode()
+                        await stream.send(f"{len(queued_frame)}\n".encode() + queued_frame)
+                        # Stream progress frames until done:true terminal frame.
+                        async with recv_ch:
+                            async for terminal in recv_ch:
+                                frame_bytes = json.dumps(terminal).encode()
+                                await stream.send(f"{len(frame_bytes)}\n".encode() + frame_bytes)
+                                if terminal.get("done"):
+                                    break
+                    else:
+                        position = await writer_queue.enqueue(writer_req)
+                        resp = {"ok": True, "queued": True, "position": position}
+                        await stream.send(json.dumps(resp).encode() + b"\n")
+                    return
                 elif op == "reindex":
+                    # Step 2.3 (B1) — enqueue; the drain is the only backend consumer.
+                    # The handler NEVER calls backend_ref() (B1 invariant).
                     root = req.get("root")
-                    from_sha = req.get("from")
+                    from_sha = req.get("from")  # may be None (W3 — server resolves at drain)
                     to_sha = req.get("to")
                     dialect = req.get("dialect")
-                    if not root or not from_sha or not to_sha:
-                        resp = {"error": "reindex op requires root, from, to"}
+                    wait = req.get("wait", False)
+                    requested_by = req.get("requested_by", "cli")
+                    if not root:
+                        resp = {"error": "reindex op requires root"}
+                        await stream.send(json.dumps(resp).encode() + b"\n")
+                        return
+                    writer_req = WriterRequest(
+                        op="reindex",
+                        root=root,
+                        dialect=dialect,
+                        from_sha=from_sha,
+                        to_sha=to_sha,
+                        requested_by=requested_by,
+                    )
+                    if wait:
+                        send_ch, recv_ch = anyio.create_memory_object_stream(max_buffer_size=64)
+                        writer_req._waiters.append(send_ch)
+                        position = await writer_queue.enqueue(writer_req)
+                        queued_frame = json.dumps(
+                            {"ok": True, "done": False, "queued": True, "position": position}
+                        ).encode()
+                        await stream.send(f"{len(queued_frame)}\n".encode() + queued_frame)
+                        async with recv_ch:
+                            async for terminal in recv_ch:
+                                frame_bytes = json.dumps(terminal).encode()
+                                await stream.send(f"{len(frame_bytes)}\n".encode() + frame_bytes)
+                                if terminal.get("done"):
+                                    break
                     else:
-                        from sqlcg.indexer.indexer import Indexer
-                        db = backend_ref()
-                        if db is None:
-                            resp = {"error": "backend not available"}
-                        else:
-                            indexer = Indexer()
-                            def _do_reindex() -> dict:
-                                return indexer.resync_changed(
-                                    _Path(root),
-                                    from_sha,
-                                    to_sha,
-                                    db,
-                                    dialect,
-                                )
-                            async with backend_lock:
-                                # R1: run off event-loop thread; R2: lock serialises
-                                summary = await _to_thread.run_sync(_do_reindex)
-                            resp = {"ok": True, "summary": summary}
+                        position = await writer_queue.enqueue(writer_req)
+                        resp = {"ok": True, "queued": True, "position": position}
+                        await stream.send(json.dumps(resp).encode() + b"\n")
+                    return
                 elif op == "query":
-                    # Framed op (v1.2.0): read-only Cypher query over the socket.
+                    # Framed op (v1.2.0): read-only SQL query over the socket.
                     # Must only be called with a framed request (sniff above sets framed=True).
-                    cypher = req.get("cypher", "")
+                    # Accept both "cypher" (legacy field name) and "sql" keys.
+                    sql = req.get("sql") or req.get("cypher", "")
                     params = req.get("params") or {}
-                    if not _is_read_only_cypher(cypher):
+                    if not _is_read_only_sql(sql):
                         resp = {"error": "query op is read-only"}
                     else:
                         db = backend_ref()
@@ -266,11 +344,11 @@ async def _control_socket_task(
                         else:
                             def _do_query() -> list:
-                                return db.run_read(cypher, params)
+                                return db.run_read(sql, params)
                             async with backend_lock:
                                 # R1: run off event-loop thread; R2: lock serialises
-                                # reads and writes on the single Kuzu connection.
+                                # reads and writes on the single DuckDB connection.
                                 rows = await _to_thread.run_sync(_do_query)
                             resp = {"ok": True, "rows": rows}
@@ -297,36 +375,37 @@ async def _control_socket_task(
 async def _stop_watcher(
     stop_event: "anyio.Event",
     db_path: "Path",
+    backend_lock: "anyio.Lock",
+    shutdown_requested: "anyio.Event",
 ) -> None:
     """Wait for stop_event then perform graceful shutdown.
-    When the control socket ``stop`` op fires ``stop_event``, this task:
-    1. Shuts down the backend (flush pending writes).
-    2. Removes the control files (.sock, .pid).
-    3. Calls ``os._exit(0)`` to terminate the process immediately.
-    We use ``os._exit(0)`` rather than a cancel-scope approach because the
-    MCP ``stdio_server`` runs stdin readline in a thread via
-    ``anyio.to_thread.run_sync`` with ``abandon_on_cancel=False`` (the
-    default).  When stdin is a pipe, the read-end cannot be closed from within
-    the subprocess to interrupt the blocking readline — the parent holds the
-    write end open.  ``os._exit`` bypasses the thread-drain wait entirely and
-    exits the process without calling ``atexit`` handlers or running
-    ``finally`` blocks in other tasks.
-    R8 ordering: backend is shut down HERE (before os._exit), not in main().
-    The ``finally`` block in ``main()`` will also try to shutdown/cleanup but
-    ``os._exit`` prevents it from running — so we do it explicitly here.
+    Shutdown ordering:
+      1. Set shutdown_requested so the drain loop exits cleanly after its
+         current drain completes (no new drains start once this is set).
+      2. Acquire backend_lock — waits until any active drain has finished
+         (committed its transaction).
+      3. Call shutdown_backend() under the lock.
+      4. Release backend_lock.
+      5. Remove control files.
+      6. Call os._exit(0).
+    We use ``os._exit(0)`` because the MCP ``stdio_server`` blocks on a pipe
+    read (``anyio.to_thread.run_sync`` with ``abandon_on_cancel=False``).
+    We cannot interrupt it without killing the process.
     """
     import sqlcg.server.tools as _tools
     from sqlcg.server.control import cleanup_control_files
     await stop_event.wait()
-    # Graceful teardown before hard exit
-    try:
-        _tools.shutdown_backend()
-    except Exception:
-        pass
+    # Signal drain loop to stop after current drain completes.
+    shutdown_requested.set()
+    # Wait for any active drain to finish before closing the backend.
+    async with backend_lock:
+        try:
+            _tools.shutdown_backend()
+        except Exception:
+            pass
     try:
         cleanup_control_files(db_path)
     except Exception:
@@ -355,9 +434,10 @@ async def _sigterm_watcher(
 async def _run_with_control(db_path: "Path", start_time: float) -> None:
     """Run the stdio MCP loop and the control-socket task in a shared TaskGroup.
-    Stop mechanism (R8 teardown ordering):
+    Stop mechanism (B2 teardown ordering):
     - Control socket ``stop`` op → ``stop_event.set()`` → ``_stop_watcher``
-      shuts down backend + removes control files + calls ``os._exit(0)``.
+      sets shutdown_requested, acquires backend_lock (waits for active drain),
+      shuts down backend, removes control files, calls ``os._exit(0)``.
     - External SIGTERM → ``_sigterm_watcher`` → same path via ``stop_event``.
     - Normal EOF on stdin (editor closes connection) → stdio loop returns →
       ``tg.cancel_scope.cancel()`` → tasks cancelled → ``main()`` finally
@@ -368,19 +448,35 @@ async def _run_with_control(db_path: "Path", start_time: float) -> None:
     with ``abandon_on_cancel=False``).  We cannot interrupt it without
     killing the process; ``_stop_watcher`` does cleanup first.
-    ``backend_lock`` is created once here and passed into
-    ``_control_socket_task`` so concurrent control ops (reindex, query) are
-    serialised behind a single lock on the Kuzu connection (R2).
+    ``backend_lock`` is created once here and passed into both
+    ``_control_socket_task`` and the ``drain_loop`` task so that:
+    - concurrent control ops (reindex, query) are serialised (R2), and
+    - _stop_watcher can acquire the lock to wait for an active drain (B2).
     """
     import anyio
     import sqlcg.server.tools as _tools
+    from sqlcg.server.writer import WriterQueue, drain_loop
     stop_event = anyio.Event()
-    backend_lock = anyio.Lock()  # R2: serialise all backend ops (Kuzu not thread-safe)
+    shutdown_requested = anyio.Event()
+    backend_lock = anyio.Lock()  # R2 + B2: serialise all backend ops
+    # Inject metrics into the queue so coalesce/drain events are persisted.
+    writer_queue = WriterQueue(metrics=_tools._metrics)
+    db_path_str = str(db_path)
     async with anyio.create_task_group() as tg:
         if sys.platform != "win32":
+            # Drain task: consumes WriterQueue; sole backend consumer (B1).
+            tg.start_soon(
+                drain_loop,
+                writer_queue,
+                db_path_str,
+                backend_lock,
+                shutdown_requested,
+            )
             # Spawn control socket alongside the stdio loop.
             tg.start_soon(
                 _control_socket_task,
@@ -389,9 +485,10 @@ async def _run_with_control(db_path: "Path", start_time: float) -> None:
                 stop_event,
                 backend_lock,
                 start_time,
+                writer_queue,
             )
             # Watch stop_event; shuts down and calls os._exit(0).
-            tg.start_soon(_stop_watcher, stop_event, db_path)
+            tg.start_soon(_stop_watcher, stop_event, db_path, backend_lock, shutdown_requested)
             # Watch for SIGTERM; fires stop_event for same clean path.
             tg.start_soon(_sigterm_watcher, stop_event)
@@ -404,7 +501,7 @@ def main(db_path: str | None = None) -> None:
     """Start the MCP server.
     Args:
-        db_path: Path to KùzuDB database. If None, uses SQLCG_DB_PATH env var
+        db_path: Path to DuckDB database. If None, uses SQLCG_DB_PATH env var
                 or ~/.sqlcg/graph.db (via get_db_path in tools module).
     """
     import time
@@ -457,5 +554,6 @@ if TYPE_CHECKING:
     import anyio
     from sqlcg.core.graph_db import GraphBackend
+    from sqlcg.server.writer import WriterQueue
 from sqlcg.server.control import sock_path  # noqa: E402  (used in _control_socket_task)

sqlcg/server/skill.py CHANGED Viewed

@@ -82,7 +82,7 @@ _WORKFLOWS = """\
 TOOL_RETURN_MODELS: dict[str, type[BaseModel]] = {
     # Operational tools — return plain dict, no Judgement
     "index_repo": BaseModel,
-    "execute_cypher": BaseModel,
+    "execute_sql": BaseModel,
     "submit_feedback": BaseModel,
     # Lineage / dependency facts
     "trace_column_lineage": LineageResult,
@@ -109,7 +109,7 @@ TOOL_RETURN_MODELS: dict[str, type[BaseModel]] = {
 _TOOL_PURPOSE: dict[str, str] = {
     "index_repo": "Index a SQL repository into the graph",
-    "execute_cypher": "Execute a read-only Cypher query against the graph",
+    "execute_sql": "Execute a read-only SQL query against the graph (DuckDB)",
     "submit_feedback": "Submit feedback (TP/FP/FN) on a tool result",
     "trace_column_lineage": "Trace upstream column lineage to its source",
     "find_table_usages": "Find all queries that consume a given table",

sql-code-graph 1.2.2__py3-none-any.whl → 1.4.0__py3-none-any.whl

sql-code-graph 1.2.2py3-none-any.whl → 1.4.0py3-none-any.whl