databricks-tellr-app 0.4.3.dev21__tar.gz → 0.4.3.dev23__tar.gz

{databricks_tellr_app-0.4.3.dev21 → databricks_tellr_app-0.4.3.dev23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: databricks-tellr-app
-Version: 0.4.3.dev21
+Version: 0.4.3.dev23
 Summary: Tellr application package for Databricks Apps
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{databricks_tellr_app-0.4.3.dev21 → databricks_tellr_app-0.4.3.dev23}/databricks_tellr_app.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: databricks-tellr-app
-Version: 0.4.3.dev21
+Version: 0.4.3.dev23
 Summary: Tellr application package for Databricks Apps
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{databricks_tellr_app-0.4.3.dev21 → databricks_tellr_app-0.4.3.dev23}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "databricks-tellr-app"
-version = "0.4.3.dev21"
+version = "0.4.3.dev23"
 description = "Tellr application package for Databricks Apps"
 readme = "README.md"
 requires-python = ">=3.10"

{databricks_tellr_app-0.4.3.dev21 → databricks_tellr_app-0.4.3.dev23}/src/api/services/chat_service.py

@@ -14,7 +14,7 @@ import re
 import threading
 import uuid
 from datetime import datetime
-from typing import Any, Dict, Generator, List, Optional
+from typing import Any, Callable, Dict, Generator, List, Optional
 
 from langchain_community.chat_message_histories import ChatMessageHistory
 from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
@@ -51,6 +51,18 @@ logger = logging.getLogger(__name__)
 #: own CSS styles it, and any content here would be content nobody asked for.
 BLANK_SLIDE_HTML = '<div class="slide"></div>'
 
+# The deck-spec entry a newly inserted slide gets: contiguous with its neighbours
+# (`DeckSpec.slide_at` depends on that) and deliberately undescribed.  The route
+# that inserts fires the spec-dirty marker and the arc-review sweeper writes the
+# prose; inventing a purpose here would put words in the architect's mouth.
+_BLANK_DECK_SPEC_ENTRY: Dict[str, Any] = {
+    "purpose": "",
+    "content_brief": "",
+    "assumes": "",
+    "hands_off": "",
+    "data_references": [],
+}
+
 
 def resolve_active_design_system_id(session_id: Optional[str]) -> Optional[int]:
     """The session's pinned/active design-system id, or None.
@@ -1117,6 +1129,7 @@ class ChatService:
                 session_id,
                 message,
                 is_first_message=is_first_message,
+                request_id=request_id,
             )
             return
 
@@ -1788,6 +1801,7 @@ class ChatService:
         message: str,
         *,
         is_first_message: bool = False,
+        request_id: Optional[str] = None,
     ) -> Generator[StreamEvent, None, None]:
         """Run one turn on the LangGraph engine, yielding events as they arrive.
 
@@ -1835,6 +1849,14 @@ class ChatService:
                 ``architect_node`` is the sole resolver of the design contract
                 and the template bytes.
             is_first_message: Whether to generate a session title this turn.
+            request_id: The async transport's ``ChatRequest.request_id``, handed
+                to ``invoke_graph`` so a node persisting a chat message tags the
+                row with it.  ``GET /chat/poll`` reads assistant text through
+                ``get_messages_for_request``, which filters on that column, so
+                without it the architect's reply is written but no polling client
+                can see it — and polling is the transport the deployed app uses.
+                ``None`` on the SSE path, which persists no request row: that
+                client is reading the yielded events instead.
 
         Yields:
             Every ``StreamEvent`` the graph emits, then ``COMPLETE`` carrying
@@ -1858,6 +1880,7 @@ class ChatService:
                     session_id,
                     {"architect_message": message},
                     emitter=event_queue,
+                    request_id=request_id,
                 )
             except Exception as e:
                 logger.error(
@@ -3343,6 +3366,13 @@ class ChatService:
         )
         self._record_deck_version(session_id, save_result)
 
+        # The deck spec's slide list has to move with the deck's.  AFTER
+        # save_slide_deck, never before: the spec write bumps deck.version, which
+        # would make the caller's expected_version stale and 409 a legitimate save.
+        spec_result = self._reorder_deck_spec_slides(session_id, new_order)
+        if spec_result:
+            self._record_deck_version(session_id, spec_result)
+
         # Create save point
         try:
             self.create_save_point(
@@ -3518,6 +3548,13 @@ class ChatService:
         )
         self._record_deck_version(session_id, save_result)
 
+        # The deck spec's slide list has to move with the deck's.  AFTER
+        # save_slide_deck, never before: the spec write bumps deck.version, which
+        # would make the caller's expected_version stale and 409 a legitimate save.
+        spec_result = self._duplicate_deck_spec_slide(session_id, index)
+        if spec_result:
+            self._record_deck_version(session_id, spec_result)
+
         # Create save point
         try:
             self.create_save_point(
@@ -3660,33 +3697,66 @@ class ChatService:
         deck_dict, _ = self._substitute_images_for_response(deck_dict, session_id=session_id)
         return deck_dict
 
-    def _insert_deck_spec_slide(
+    def _rewrite_deck_spec_slides(
         self,
         session_id: str,
-        position: int,
+        reorder: Callable[[List[Dict[str, Any]]], Optional[List[Dict[str, Any]]]],
+        *,
+        label: str,
     ) -> Optional[Dict[str, Any]]:
-        """Give the deck spec an entry at *position* and shift the entries above it.
+        """Re-order the deck spec's slide entries and re-stamp their positions.
 
         A deck spec describes the deck slide by slide, keyed on ``position``
         (``SlideSpec.position`` is the canonical identity, looked up by
-        ``DeckSpec.slide_at`` and never by list index).  Inserting a slide without
-        renumbering would leave every entry above the insertion point describing
-        its neighbour.
-
-        The new entry is a PLACEHOLDER — empty purpose and brief.  The route that
-        reaches this fires the spec-dirty marker, and the arc-review sweeper is
-        what actually describes the new slide; inventing a purpose here would put
-        words in the architect's mouth, and leaving the entry out entirely would
-        break the contiguity ``slide_at`` depends on.
+        ``DeckSpec.slide_at`` and never by list index).  Any mutation that moves,
+        adds or removes a slide must move its spec entry with it, or every entry
+        past the change describes its neighbour.
+
+        Before this existed, ``deck_spec_json`` was renumbered on INSERT and on
+        nothing else (final review C1), so after a delete, a duplicate or a reorder
+        the deck-level spec and the per-row ``deck_spec_slide`` fragments — two
+        representations of the same thing — disagreed, and three consumers read the
+        deck-level one: the builder brief, §4.6's re-review and ``architect_node``'s
+        edit-turn fallback.  Measured: an edit turn dispatched a builder for a
+        position with no slide and the deck gained one nobody asked for.
+
+        WHY ONE PRIMITIVE AND NOT FOUR WRITERS
+        --------------------------------------
+        Four hand-copied row-writers diverging is the defect class the previous
+        workstream's final review found, and C1 is itself an instance of it — insert
+        renumbered, its three siblings did not.  So the four routes supply only
+        *which entries end up in which order*; the position stamping, the raw-dict
+        handling, the author stamp and the failure policy live here once.
+
+        *reorder* receives the positioned entries **in position order** and returns
+        the new list in the new order — it does not touch ``position`` at all, which
+        this method re-stamps contiguously from 0.  Returning ``None`` leaves the
+        spec untouched.
+
+        Operates on the RAW spec dict rather than parsing a ``DeckSpec``: a spec that
+        no longer validates (an older shape, a hand-edited column) must not be
+        destroyed by a renumber, and nothing here reads a field other than
+        ``position``.  Entries that are not dicts, or whose ``position`` is not an
+        int, cannot be identified with a slide, so they are carried through
+        **unchanged and last** rather than silently permuted.
+
+        A spec write that fails must never fail the mutation: the slide change is
+        already committed by the time this runs, and the route's spec-dirty marker
+        still fires, so the sweeper re-describes the deck either way.
 
-        Operates on the RAW spec dict rather than parsing a ``DeckSpec``: a spec
-        that no longer validates (an older shape, a hand-edited column) must not be
-        destroyed by a renumber, and this only touches ``slides[*].position``.
+        Args:
+            session_id: The mutating session.  Resolves to the deck owner in the
+                layers below, exactly as every other mutation does.
+            reorder: Takes the positioned entries in position order, returns them in
+                their new order, or ``None`` to leave the spec alone.  Raises
+                ``IndexError`` when the spec cannot answer the mutation.
+            label: Route name, for the log lines only.
 
         Returns:
             The writer's result dict (carrying the new deck ``version``), or None
-            when there was no spec to shift — a deck built before the spec existed
-            is not an error.
+            when there was no spec to renumber, the spec was unreadable, or
+            *reorder* declined — none of which is an error.  A deck built before the
+            spec column existed has no spec.
         """
         # Imported here, not at module scope: src.api.services.__init__ imports
         # chat_service, so a module-level import of a sibling service is circular.
@@ -3698,7 +3768,7 @@ class ChatService:
         try:
             spec = read_deck_spec(session_id)
         except Exception as e:
-            logger.warning(f"Failed to read deck spec (insert_slide): {e}")
+            logger.warning(f"Failed to read deck spec ({label}): {e}")
             return None
 
         if not spec:
@@ -3708,37 +3778,48 @@ class ChatService:
         if not isinstance(slides, list):
             logger.warning(
                 "deck_spec_json has no slides list; leaving it untouched",
-                extra={"session_id": session_id},
+                extra={"session_id": session_id, "route": label},
             )
             return None
 
-        shifted: List[Dict[str, Any]] = []
+        # One pass, and identity is never inferred from equality: two entries with
+        # the same content are still two entries.
+        positioned: List[Dict[str, Any]] = []
+        unpositioned: List[Any] = []
         for entry in slides:
-            if not isinstance(entry, dict):
-                shifted.append(entry)
-                continue
+            if isinstance(entry, dict) and isinstance(entry.get("position"), int):
+                positioned.append(entry)
+            else:
+                unpositioned.append(entry)
+        positioned.sort(key=lambda e: e["position"])
+
+        try:
+            rewritten = reorder(list(positioned))
+        except IndexError:
+            # The route validated its index against the DECK, so an index the spec
+            # cannot answer means the two have already drifted.  Leaving a drifted
+            # spec alone is recoverable; renumbering it against the wrong slides is
+            # not.
+            logger.warning(
+                "deck spec has %s entries and cannot answer this mutation; "
+                "leaving it untouched",
+                len(positioned),
+                extra={"session_id": session_id, "route": label},
+            )
+            return None
+
+        if rewritten is None:
+            return None
+
+        renumbered: List[Any] = []
+        for index, entry in enumerate(rewritten):
             entry = dict(entry)
-            entry_position = entry.get("position")
-            if isinstance(entry_position, int) and entry_position >= position:
-                entry["position"] = entry_position + 1
-            shifted.append(entry)
-
-        shifted.append(
-            {
-                "position": position,
-                "purpose": "",
-                "content_brief": "",
-                "assumes": "",
-                "hands_off": "",
-                "data_references": [],
-            }
-        )
-        shifted.sort(
-            key=lambda e: e.get("position", 0) if isinstance(e, dict) else 0
-        )
+            entry["position"] = index
+            renumbered.append(entry)
+        renumbered.extend(unpositioned)
 
         spec = dict(spec)
-        spec["slides"] = shifted
+        spec["slides"] = renumbered
 
         try:
             _user = get_current_username()
@@ -3752,11 +3833,105 @@ class ChatService:
                 modified_by=_user,
             )
         except Exception as e:
-            # The slide is already saved and the marker will still fire, so a spec
-            # write that fails must not fail the insert.
-            logger.warning(f"Failed to shift deck spec (insert_slide): {e}")
+            logger.warning(f"Failed to write deck spec ({label}): {e}")
             return None
 
+    def _insert_deck_spec_slide(
+        self,
+        session_id: str,
+        position: int,
+    ) -> Optional[Dict[str, Any]]:
+        """Give the deck spec a placeholder entry at *position*, shifting the rest up.
+
+        *position* is where the slide ACTUALLY landed, which the caller computes
+        before the insert: ``SlideDeck.insert_slide`` delegates to ``list.insert``,
+        which clamps a too-large position into an append, so the requested position
+        and the real one diverge on an append.  The clamp is re-applied here against
+        the spec's own length rather than trusted, because a spec shorter than the
+        deck would otherwise leave a hole.
+        """
+
+        def _splice(entries: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+            landed = min(position, len(entries))
+            return (
+                entries[:landed]
+                + [dict(_BLANK_DECK_SPEC_ENTRY)]
+                + entries[landed:]
+            )
+
+        return self._rewrite_deck_spec_slides(
+            session_id, _splice, label="insert_slide"
+        )
+
+    def _delete_deck_spec_slide(
+        self,
+        session_id: str,
+        index: int,
+    ) -> Optional[Dict[str, Any]]:
+        """Drop the deleted slide's spec entry and shift every higher entry down."""
+
+        def _drop(entries: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+            if index >= len(entries):
+                raise IndexError(index)
+            return entries[:index] + entries[index + 1 :]
+
+        return self._rewrite_deck_spec_slides(
+            session_id, _drop, label="delete_slide"
+        )
+
+    def _duplicate_deck_spec_slide(
+        self,
+        session_id: str,
+        index: int,
+    ) -> Optional[Dict[str, Any]]:
+        """Copy the duplicated slide's spec entry in beside it.
+
+        The clone gets its SOURCE's brief, not a blank one: a duplicate is a copy of
+        a slide we already have a description for, so that description is known to be
+        right, where insert's newcomer has none.  The route's spec-dirty marker fires
+        either way, so the sweeper re-describes both — this decides which starting
+        point is true, not who writes the final prose.
+        """
+
+        def _clone(entries: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+            if index >= len(entries):
+                raise IndexError(index)
+            return (
+                entries[: index + 1]
+                + [dict(entries[index])]
+                + entries[index + 1 :]
+            )
+
+        return self._rewrite_deck_spec_slides(
+            session_id, _clone, label="duplicate_slide"
+        )
+
+    def _reorder_deck_spec_slides(
+        self,
+        session_id: str,
+        new_order: List[int],
+    ) -> Optional[Dict[str, Any]]:
+        """Permute the spec entries by the same permutation the slides took.
+
+        ``new_order[j]`` is the OLD index of the slide that now sits at *j* — the
+        route validates it is a true permutation of ``range(len(slides))`` before
+        reaching here.
+
+        This is the case the consumer-side guard structurally cannot cover: a reorder
+        preserves the position SET, so ``_persisted_spec_describes_these_rows`` reads
+        a reordered deck as aligned while every entry describes a different slide.
+        Renumbering in the route is the only thing that closes it.
+        """
+
+        def _permute(entries: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+            if len(new_order) != len(entries):
+                raise IndexError(len(new_order))
+            return [entries[old] for old in new_order]
+
+        return self._rewrite_deck_spec_slides(
+            session_id, _permute, label="reorder_slides"
+        )
+
     def delete_slide(self, session_id: str, index: int, *, expected_version: Optional[int] = None) -> Dict[str, Any]:
         """Delete a slide.
 
@@ -3799,6 +3974,13 @@ class ChatService:
         )
         self._record_deck_version(session_id, save_result)
 
+        # The deck spec's slide list has to move with the deck's.  AFTER
+        # save_slide_deck, never before: the spec write bumps deck.version, which
+        # would make the caller's expected_version stale and 409 a legitimate save.
+        spec_result = self._delete_deck_spec_slide(session_id, index)
+        if spec_result:
+            self._record_deck_version(session_id, spec_result)
+
         # Create save point
         try:
             self.create_save_point(

{databricks_tellr_app-0.4.3.dev21 → databricks_tellr_app-0.4.3.dev23}/src/services/graph/builder.py

@@ -73,7 +73,10 @@ from langgraph.graph import END, START, StateGraph
 from src.core.checkpointer import get_checkpointer
 from src.core.user_context import get_current_user
 from src.services.foreman_service import CAP
-from src.services.graph.event_emitter import set_event_emitter
+from src.services.graph.event_emitter import (
+    set_chat_request_id,
+    set_event_emitter,
+)
 from src.services.graph.nodes import (
     architect_node,
     build_reviewer_node,
@@ -188,6 +191,7 @@ def invoke_graph(
     emitter: Any = None,
     principal: Optional[str] = None,
     describe_only: bool = False,
+    request_id: Optional[str] = None,
 ) -> Dict[str, Any]:
     """Run one turn of the graph for *session_id*.
 
@@ -210,6 +214,14 @@ def invoke_graph(
             arc-review sweeper passes it: that turn exists to re-describe the
             deck a human hand-edited, and dispatching builders would overwrite
             the very edits that scheduled it.
+        request_id: The ``ChatRequest.request_id`` this turn is answering, or
+            ``None``.  Bound in a ``ContextVar`` so a node persisting a chat
+            message can TAG it: ``GET /chat/poll`` reads a turn's assistant text
+            through ``get_messages_for_request``, which filters on that column,
+            so an untagged row is invisible to every polling client — the
+            transport the deployed app uses.  ``None`` is correct on the SSE path
+            (no request row exists) and on the sweeper (no request at all); those
+            rows are still durable and still reach ``_conversation``.
 
     ``describe_only`` is turn-scoped, and it is wrapped HERE because this is the
     only place that knows ``turn_id``.  It is written on **every** invocation,
@@ -231,12 +243,17 @@ def invoke_graph(
 
     The emitter is set on every invocation, ``None`` included: that is the reset
     a turn resumed in a new process needs, or events queue into the first
-    process's queue forever.
+    process's queue forever.  ``request_id`` is set unconditionally for the same
+    reason, and the failure it prevents is worse than a lost event: a stale id
+    left in the var would tag turn 2's chat rows with turn 1's request, so turn
+    1's poll would receive turn 2's reply and turn 2's own poll would never see
+    it.
     """
     turn_id = uuid.uuid4().hex
     initiated_by = principal or get_current_user()
 
     set_event_emitter(emitter)
+    set_chat_request_id(request_id)
 
     state: Dict[str, Any] = dict(initial or {})
     state.update(

{databricks_tellr_app-0.4.3.dev21 → databricks_tellr_app-0.4.3.dev23}/src/services/graph/event_emitter.py

@@ -69,6 +69,46 @@ slide_cursor_var: ContextVar[Optional[list]] = ContextVar(
     "graph_slide_cursor", default=None
 )
 
+#: The ``ChatRequest.request_id`` this turn is running under, or ``None``.
+#:
+#: **This is not an emission concern, and it is here because the LIFECYCLE is the
+#: same one.**  ``GET /chat/poll`` reads a turn's assistant text out of
+#: ``get_messages_for_request``, which filters ``SessionMessage.request_id ==
+#: request_id`` — so a chat row written by a node with no request id is invisible
+#: to every polling client, which is the transport the deployed app uses.  The id
+#: cannot travel in ``GraphState`` (undeclared keys are silently dropped and
+#: declaring it would put a transport detail in the graph's contract) and it
+#: cannot be re-derived inside a node, so it rides the same ``ContextVar``
+#: mechanism the emitter does, is set by ``invoke_graph`` on **every** invocation,
+#: and is ``None`` on the SSE path (which has no request row) and on the sweeper.
+chat_request_id_var: ContextVar[Optional[str]] = ContextVar(
+    "graph_chat_request_id", default=None
+)
+
+
+def set_chat_request_id(request_id: Optional[str]) -> None:
+    """Bind *request_id* for this turn's context; ``None`` clears it.
+
+    Called by ``invoke_graph`` unconditionally, for the reason
+    :func:`set_event_emitter` is: a value left over from an earlier turn in the
+    same process would tag turn 2's rows with turn 1's request, and turn 1's
+    poll — which a client may still be draining — would then receive turn 2's
+    reply while turn 2's own poll never saw it.
+    """
+    chat_request_id_var.set(request_id)
+
+
+def get_chat_request_id() -> Optional[str]:
+    """The request id this turn runs under, or ``None`` when there is no request.
+
+    ``None`` is a normal, correct answer on two paths: the SSE transport (which
+    persists no ``ChatRequest`` row at all) and the arc-review sweeper (which has
+    no request). A row written with ``None`` is still a durable transcript row and
+    still reaches ``_conversation``; it is only the *polling* projection that
+    needs the id.
+    """
+    return chat_request_id_var.get()
+
 
 def set_event_emitter(emitter: Optional[queue.Queue]) -> None:
     """Install *emitter* as this context's event queue, and reset the slide cursor.