pi-ghost-autocomplete 0.6.0 → 0.7.0

package/dist/intent.d.ts

@@ -39,6 +39,31 @@
  *                → LLM only (skip deterministic)
  */
 export type PromptIntent = "command" | "question" | "instruction" | "path-ref" | "prose";
+/**
+ * Coarse communication register — *what kind of text* the user is composing,
+ * independent of the syntactic {@link PromptIntent}. Drives provider routing
+ * (FIM vs chat) and prompt persona (code-edit vs agent-message vs general
+ * prose) so general prose no longer gets the "message to a coding agent"
+ * framing that distorts non-English / non-instructional text.
+ *
+ * - `code`           — shell commands, file paths, code-shaped drafts.
+ *                      Routed to FIM; deterministic path completion active.
+ * - `agent-message`  — a request/instruction/question addressed to the AI
+ *                      assistant. Routed to chat with the "continue the
+ *                      developer's message" persona.
+ * - `general-prose`  — natural-language writing that is NOT an instruction to
+ *                      an agent (notes, documentation, free text). Routed to
+ *                      chat but with a neutral "continue the text" persona
+ *                      that preserves language/voice without reframing it as
+ *                      a task.
+ *
+ * Separated from `PromptIntent` so the two decisions (syntactic form vs
+ * communication target) can evolve independently. E.g. a German imperative
+ * ("Bitte ändere …") and English free prose ("I was thinking …") both land
+ * on the chat leg, but the former is an agent-message while the latter is
+ * general-prose — producing different system prompts.
+ */
+export type CompletionRegister = "code" | "agent-message" | "general-prose";
 /**
  * Action verbs that strongly signal the user is about to reference a file
  * path. Used by `isPathContext`, `draftLooksLikePathContext`, and the
@@ -58,8 +83,12 @@ export declare const ACTION_VERBS_ALT: string;
 /**
  * Heuristic: does the text before the cursor look like natural-language prose
  * rather than a command / path context? Returns true when there are 4+ word
- * tokens (\w{2,}) — a strong signal the user is writing a sentence, not a
- * filename.
+ * tokens — a strong signal the user is writing a sentence, not a filename.
+ *
+ * Uses Unicode-aware tokenization (session 52) so accented characters
+ * (German umlauts, French diacritics) count as part of the word instead of
+ * fragmenting it. The pre-52 `[A-Za-z]{2,}` regex turned "möchte" into the
+ * spurious token "chte", miscounting words and misclassifying prose.
  *
  * This prevents the fuzzy matcher from proposing `config.ts` when the user
  * types "please add a con|" ("con" fuzzy-matches config.ts at score ~75).
@@ -123,6 +152,29 @@ export declare function isPathContext(lineBeforeCursor: string, _token: {
  * providers use these so the wording stays consistent.
  */
 export declare const INTENT_DESCRIPTION: Record<PromptIntent, string>;
+/**
+ * Persona/framing instruction per {@link CompletionRegister}. Injected into
+ * the chat system prompt so the model knows *what* it is continuing:
+ *
+ * - `code`          — framed as code/command completion (FIM uses its own
+ *                     intent header, so this is mainly for chat providers
+ *                     that receive a code-shaped request).
+ * - `agent-message` — "continue the user's message to an AI assistant,
+ *   don't answer it."
+ * - `general-prose` — "continue the text naturally in the same language
+ *   and voice" — NO coding-agent framing, so general prose isn't distorted
+ *   into a task instruction.
+ */
+export declare const REGISTER_DESCRIPTION: Record<CompletionRegister, string>;
+/**
+ * Derive the {@link CompletionRegister} from a classified intent. Pure —
+ * used by the editor, router, and providers so all three agree on the
+ * register without recomputing it.
+ *
+ * Unclassified (`undefined`) intent defaults to `code` to preserve the
+ * pre-52 default for the common cursor-at-end-of-code case.
+ */
+export declare function registerFromIntent(intent: PromptIntent | undefined): CompletionRegister;
 /**
  * Classify the user's current prompt intent. Uses simple regex / keyword
  * heuristics to decide which completion pipeline to activate.
@@ -148,9 +200,12 @@ export declare function classifyPromptIntent(lineBeforeCursor: string, previousL
 export type ProseReason = ProseContextReason | "single-line";
 /** Detailed intent classification including the prose-rule reason when
  *  applicable. Lets the editor log *why* the deterministic path was
- *  suppressed without re-running the heuristic. */
+ *  suppressed without re-running the heuristic. Includes the derived
+ *  {@link CompletionRegister} so callers (router, providers) don't need to
+ *  recompute it. */
 export interface IntentClassification {
     intent: PromptIntent;
+    register: CompletionRegister;
     proseReason?: ProseReason;
 }
 /**

package/dist/metrics.d.ts

@@ -19,18 +19,32 @@ export type RejectionReason = "stale-text" | "newline" | "too-long" | "assistant
  * whose output the sanitizer dropped, or a request that never reached the
  * backend — that distinction previously needed synthetic probing.
  *
- *  - `model-empty`   — the model returned empty/whitespace-only text (the
- *                      Mercury Edit "nothing to insert" case from session 51).
- *  - `sanitizer-null`— the model returned content but `sanitizeCompletion`
- *                      rejected it (echo / filename-in-prose / markdown).
- *  - `http-error`    — the request failed: missing key, non-2xx, malformed
- *                      JSON, fetch throw, or timeout.
+ *  - `model-empty`             — the model returned empty/whitespace-only text
+ *                               (the Mercury Edit "nothing to insert" case
+ *                               from session 51).
+ *  - `token-budget-exhausted`  — the provider spent its entire `max_tokens`
+ *                               budget before emitting any visible content.
+ *                               Observed with Mercury 2 chat, which consumes
+ *                               budget on internal reasoning tokens first:
+ *                               `finish_reason="length"` with `content=null`
+ *                               and `completion_tokens=0`. A capacity /
+ *                               configuration issue, not a model decline.
+ *  - `sanitizer-null`          — the model returned content but
+ *                               `sanitizeCompletion` rejected it (echo /
+ *                               filename-in-prose / markdown).
+ *  - `response-schema-error`  — the response parsed as JSON but lacked the
+ *                               expected `choices[].message` shape (empty
+ *                               `choices`, missing `message`). Indicates a
+ *                               backend/schema mismatch rather than an empty
+ *                               model verdict.
+ *  - `http-error`              — the request failed: missing key, non-2xx,
+ *                               malformed JSON, fetch throw, or timeout.
  *
  * Only set by providers that implement `consumeLastEmptyReason()`. Records from
  * providers that don't (or that predate this field) omit it — bench buckets
  * those as `unattributed`.
  */
-export type EmptyReason = "model-empty" | "sanitizer-null" | "http-error";
+export type EmptyReason = "model-empty" | "token-budget-exhausted" | "sanitizer-null" | "response-schema-error" | "http-error";
 export interface ProviderLatency {
     provider: string;
     latencyMs: number;
@@ -45,6 +59,9 @@ export interface MetricsRecord {
     winner?: string;
     result: RequestResult;
     rejectionReason?: RejectionReason;
+    /** Session 52 — completion register for this request. Lets bench split
+     *  abort / latency / quality by register (code vs agent-message vs prose). */
+    register?: string;
     /** When `result === "empty"`, the diagnosed cause (session-51 todo #4). */
     emptyReason?: EmptyReason;
     completionLength?: number;

package/dist/profile-store.d.ts

@@ -47,7 +47,7 @@ export declare class ProfileStore {
      * Capture an accept (full or partial). `weight` defaults to 3 to match
      * PRD §M2 trie weights. Returns true when a record was emitted.
      */
-    captureAccept(prefix: string, completion: string, weight?: number, ts?: number): boolean;
+    captureAccept(prefix: string, completion: string, weight?: number, ts?: number, register?: string): boolean;
     /** Public lookup surfaces — thin wrappers around `profile.ts`. */
     bestSlashMatch(prefix: string): {
         cmd: string;
@@ -60,8 +60,14 @@ export declare class ProfileStore {
      * record. Returns the completion tokens + summed weight, or null when
      * the prefix has never been the exact key of a captured accept.
      * `prefix` is hashed in-process; the raw string never leaves memory.
+     *
+     * Session 52: when `register` is provided, cross-register matches are
+     * filtered. A record with a DIFFERENT register is skipped (a code-context
+     * accept shouldn't be replayed in a prose context). Records without a
+     * register (pre-session-52) are served with a reduced weight (0.7×) so
+     * they don't crowd out register-matched candidates but aren't dropped.
      */
-    lookupAcceptByPrefix(prefix: string): {
+    lookupAcceptByPrefix(prefix: string, register?: string): {
         completionTokens: string[];
         weight: number;
     } | null;

package/dist/profile.d.ts

@@ -30,6 +30,13 @@ export interface ProfileRecord {
     prefixTailTokens?: string[];
     /** `kind=accept`: tokens of the completion (cap 8). */
     completionTokens?: string[];
+    /** Session 52 — completion register at capture time (`code`,
+     *  `agent-message`, `general-prose`). When set, `lookupAcceptByPrefix`
+     *  can skip cross-register replays (a code-context accept shouldn't be
+     *  served as a provisional ghost in a prose context). Absent on records
+     *  predating session 52 — loaded as `unknown` and penalized but not
+     *  dropped. */
+    register?: string;
 }
 /**
  * In-memory aggregated view of the profile. The maps are weight-sums per
@@ -43,6 +50,7 @@ export interface ProfileIndex {
     acceptByPrefix: Map<string, {
         completionTokens: string[];
         weight: number;
+        register?: string;
     }>;
     totalAcceptRecords: number;
 }
@@ -87,7 +95,7 @@ export declare function recordsFromCommit(text: string, ts?: number): ProfileRec
  * hashed; `completion` is tokenized. The raw prefix never appears in the
  * returned record. Drops the whole record if either side fails sanitize.
  */
-export declare function recordFromAccept(prefix: string, completion: string, weight?: number, ts?: number): ProfileRecord | null;
+export declare function recordFromAccept(prefix: string, completion: string, weight?: number, ts?: number, register?: string): ProfileRecord | null;
 /**
  * Append one or more records to the JSONL file. Returns silently on write
  * failure — profile logging must never break ghost completion.

package/dist/providers/openai-compat-provider.d.ts

@@ -5,7 +5,7 @@
 import type { OpenAICompatProviderConfig } from "../config.js";
 import { type CandidateSet, type GhostProvider, type GhostRequest } from "./types.js";
 import type { EmptyReason } from "../metrics.js";
-import { type PromptIntent } from "../intent.js";
+import { type CompletionRegister, type PromptIntent } from "../intent.js";
 export interface PreflightResult {
     ok: boolean;
     status?: number;
@@ -57,7 +57,7 @@ export declare class OpenAICompatProvider implements GhostProvider {
  * `PI_GHOST_FIM_CONTEXT_ALL_INTENTS=1` to force-include context for every
  * intent (A/B against the empty-rate fix).
  */
-export declare function fimContextIncluded(intent: PromptIntent | undefined, allIntentsRaw: string | undefined): boolean;
+export declare function fimContextIncluded(intent: PromptIntent | undefined, allIntentsRaw: string | undefined, register?: CompletionRegister): boolean;
 /**
  * Build the FIM prompt/suffix pair from the ghost request.
  *

package/dist/providers/router.d.ts

@@ -4,27 +4,37 @@
  */
 /**
  * Register-routing provider (PRD §M3h). Holds two legs and dispatches each
- * request to the one whose *register* fits the user's intent:
+ * request to the one whose {@link CompletionRegister} fits the user's intent:
  *
- *  - **FIM leg** (code register) — `command`, `path-ref`, and unclassified
+ *  - **FIM leg** (`code` register) — `command`, `path-ref`, and unclassified
  *    requests. A code fill-in-the-middle model (Mercury Edit) completes
  *    commands, paths, and code-shaped drafts.
- *  - **chat leg** (instruction register) — `instruction`, `question`, `prose`.
- *    A conversational model completes the developer's *message to the AI coding
- *    agent* (their request/instruction), instead of writing the solution. This
- *    is the fix for "the autocomplete keeps suggesting solutions": a code-FIM
- *    model, fed a natural-language prompt, completes it like code.
+ *  - **chat leg** (`agent-message` + `general-prose` registers) — `instruction`,
+ *    `question`, `prose`. A conversational model completes the developer's
+ *    *message to the AI assistant* (for `agent-message`) or continues
+ *    ordinary prose naturally (for `general-prose`), instead of writing code.
+ *    This is the fix for "the autocomplete keeps suggesting solutions": a
+ *    code-FIM model, fed a natural-language prompt, completes it like code.
+ *
+ * Session 52: routing is now {@link CompletionRegister}-aware via
+ * {@link routeToChatByRegister}, falling back to intent-based routing
+ * ({@link routeToChat}) when `req.register` is absent (legacy providers).
  *
  * When `chat` is null the router degrades to FIM-only — byte-for-byte the
  * default behavior before routing existed, so enabling router mode without a
  * reachable chat backend never regresses.
  */
-import type { PromptIntent } from "../intent.js";
+import type { CompletionRegister, PromptIntent } from "../intent.js";
 import type { EmptyReason } from "../metrics.js";
 import type { PreflightResult } from "./openai-compat-provider.js";
 import { type CandidateSet, type GhostProvider, type GhostRequest } from "./types.js";
-/** Pure routing predicate — exported for direct testing. */
+/** Pure routing predicate — exported for direct testing. Legacy: routes
+ *  based on intent. Prefer {@link routeToChatByRegister} when the register
+ *  is available. */
 export declare function routeToChat(intent: PromptIntent | undefined): boolean;
+/** Pure routing predicate based on {@link CompletionRegister} (session 52).
+ *  When `register` is absent, falls back to deriving it from `intent`. */
+export declare function routeToChatByRegister(register: CompletionRegister | undefined, intent: PromptIntent | undefined): boolean;
 export declare class RouterProvider implements GhostProvider {
     private readonly fim;
     private readonly chat;
@@ -35,15 +45,30 @@ export declare class RouterProvider implements GhostProvider {
     constructor(fim: GhostProvider, chat: GhostProvider | null);
     get debounceMs(): number;
     /**
-     * Which leg an intent resolves to — `"chat"` only when a chat leg is
-     * present AND the intent is natural-language; otherwise `"fim"` (including
-     * the chat-absent fallback). Pure; used by the editor to tag metrics with
-     * the served register so bench can split FIM-vs-chat quality.
+     * Register-aware debounce (session 52). Natural-language registers get a
+     * longer debounce because the user types more deliberately and the model
+     * takes longer to respond — a short debounce wastes API calls that get
+     * aborted by the next keystroke. Code-shaped requests keep the base
+     * debounce since they're short and benefit from speed.
+     *
+     * Multipliers are overridable via env vars for A/B tuning:
+     * `PI_GHOST_DEBOUNCE_AGENT_MULT` (default 1.5),
+     * `PI_GHOST_DEBOUNCE_PROSE_MULT` (default 1.8).
+     */
+    debounceMsFor(register?: CompletionRegister): number;
+    /**
+     * Which leg a request resolves to — `"chat"` only when a chat leg is
+     * present AND the register/intent is natural-language; otherwise `"fim"`
+     * (including the chat-absent fallback). Pure; used by the editor to tag
+     * metrics with the served register so bench can split FIM-vs-chat quality.
+     *
+     * Session 52: prefers `register` over `intent` so the coarser register
+     * drives routing. Falls back to intent when register is absent.
      */
-    legFor(intent: PromptIntent | undefined): "fim" | "chat";
+    legFor(intent: PromptIntent | undefined, register?: CompletionRegister): "fim" | "chat";
     /** Choose the leg for this request. Chat leg only when present AND the
-     *  intent is natural-language; otherwise FIM. Records the choice so
-     *  `consumeLastEmptyReason` can forward to the leg that actually ran. */
+     *  intent/register is natural-language; otherwise FIM. Records the choice
+     *  so `consumeLastEmptyReason` can forward to the leg that actually ran. */
     private leg;
     /** Forward the empty-cause diagnostic to whichever leg last served a
      *  request. Null when no request has run yet or the leg doesn't track it. */

package/dist/providers/types.d.ts

@@ -3,7 +3,7 @@
  * SPDX-FileCopyrightText: 2025 Alessandro Pungitore <alessandro@pungitore.ch>
  */
 import type { PreflightResult } from "./openai-compat-provider.js";
-import type { PromptIntent } from "../intent.js";
+import type { PromptIntent, CompletionRegister } from "../intent.js";
 import type { EmptyReason } from "../metrics.js";
 export interface GhostRequest {
     editorText: string;
@@ -29,6 +29,14 @@ export interface GhostRequest {
      * classification was added).
      */
     intent?: PromptIntent;
+    /**
+     * Communication register (session 52) — derived from {@link intent} but
+     * coarser: `code` / `agent-message` / `general-prose`. The router uses it
+     * to choose the FIM vs chat leg; chat providers use it to pick the system
+     * prompt persona (agent-message vs general-prose). When absent, providers
+     * fall back to deriving it from {@link intent}.
+     */
+    register?: CompletionRegister;
 }
 /**
  * One candidate completion. `avgLogprob` is the average per-token logprob if
@@ -62,6 +70,15 @@ export interface StreamDelta {
 export interface GhostProvider {
     readonly name: string;
     readonly debounceMs: number;
+    /**
+     * Optional register-aware debounce (session 52). When present, the editor
+     * uses this instead of {@link debounceMs} so natural-language registers
+     * (agent-message, general-prose) get a longer debounce than code. This
+     * reduces wasted API calls during fast typing without hurting the UX of
+     * deliberate prose composition. When absent, callers fall back to
+     * {@link debounceMs}.
+     */
+    debounceMsFor?(register?: CompletionRegister): number;
     predict(req: GhostRequest): Promise<string | null>;
     /**
      * Optional multi-candidate variant. When absent the editor wraps