pi-simocracy 0.4.1 → 0.5.1

package/README.md

@@ -120,17 +120,36 @@ The same actions are exposed to pi as tools, so the model can drive them itself:
    - `org.simocracy.agents`  — short description + full constitution
    - `org.simocracy.style`   — speaking style / mannerisms
 4. **Render.** Fetch the sprite blob via `com.atproto.sync.getBlob`,
-   decode it, crop the front-facing idle frame, and emit as 24-bit ANSI
-   using the upper-half-block character `▀` so each terminal cell paints
-   two pixels. Two render paths depending on the sim's `spriteKind`:
+   decode it, crop the front-facing idle frame. Two render paths
+   depending on the sim's `spriteKind`:
    - **`pipoya`** (legacy + default): 128×128 PNG, 4×4 of 32×32 walking
      frames; decode with `pngjs`, take row 0 col 0 at native size.
    - **`codexPet`** (OpenAI hatch-pet output): 1536×1872 atlas, 8×9 of
      192×208 cells. PNG sheets decode through `pngjs`; WebP sheets
      decode through `@jsquash/webp` (wasm, lazy-init). The idle cell
-     (row 0 col 0) is box-downscaled to ~32 wide so the inline render
-     stays comparable in height to a pipoya sprite.
-   Transparent regions show pi's background through.
+     (row 0 col 0) is the source for both render paths below.
+
+   Then emit through one of two terminal output paths, picked
+   automatically per-terminal:
+   - **Inline graphics** (Kitty graphics protocol or iTerm2 inline
+     images). The cropped RGBA cell is re-encoded to PNG via `pngjs`
+     and handed to pi-tui's `Image` component, which transmits it as a
+     true-color bitmap. Used in Kitty, Ghostty, WezTerm, Konsole, and
+     iTerm2 — the terminal does its own scaling, so pixel-art sprites
+     stay crisp and codex pets render at full fidelity.
+   - **24-bit ANSI half-blocks** (universal fallback). Emits the
+     upper/lower half-block characters `▀`/`▄` with `\x1b[38;2;…m`
+     true-color escapes so each terminal cell paints two pixels. Used
+     in Apple Terminal, plain SSH, tmux without passthrough, and
+     anywhere that doesn't advertise inline-image support. Transparent
+     regions show pi's background through.
+
+   To force the half-block path even on a graphics-capable terminal
+   (handy for screenshots and demo recordings), set
+   `SIMOCRACY_INLINE_GRAPHICS=ansi`. The default is `auto`.
+
+   Both paths use the same upstream RGBA buffer — swapping between
+   them changes only the final encoding step, never the source pixels.
 5. **Inject.** A `before_agent_start` event handler appends the sim's
    identity + constitution + speaking style to pi's system prompt **every
    turn**. After `/sim unload`, a one-shot override fires on the next
@@ -151,6 +170,7 @@ src/
 ├── simocracy.ts      # indexer + PDS client (read-only fetchers)
 ├── writes.ts         # PDS writers + ownership / sign-in preconditions
 ├── png-to-ansi.ts    # RGBA half-block ANSI renderer + downscalers
+├── png-encode.ts     # RGBA → PNG encoder for inline-graphics protocols
 ├── webp-to-rgba.ts   # @jsquash/webp wrapper for codex pet WebP sheets
 ├── openrouter.ts     # minimal OpenRouter client (only used by simocracy_chat)
 └── auth/             # ATProto OAuth loopback flow + session storage

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-simocracy",
-  "version": "0.4.1",
+  "version": "0.5.1",
   "description": "Pi extension: load a Simocracy sim into your chat — see its pixel-art sprite render inline in the terminal and roleplay with it.",
   "type": "module",
   "author": "David Dao <david@gainforest.earth> (https://github.com/daviddao)",

package/src/animated-image.ts

@@ -0,0 +1,188 @@
+/**
+ * AnimatedImage — a pi-tui `Component` that cycles through PNG frames in
+ * place using the Kitty graphics protocol's image-by-ID swap.
+ *
+ * Why not just reuse pi-tui's `Image`?
+ *
+ * `Image` caches its render output and `Image.base64Data` is declared
+ * `private`, so we can't mutate the source bytes between renders to
+ * swap frames. Pi-tui's own animated component (`Loader`, the spinner)
+ * works because it extends `Text` and calls `setText()` to swap its
+ * payload — there's no equivalent setter on `Image`. Rather than reach
+ * into pi-tui internals or upstream a patch, we ship a tiny standalone
+ * `Component` that owns its frames + timer and emits the appropriate
+ * Kitty / iTerm2 escape on every `render()` call.
+ *
+ * Behaviour:
+ *
+ *  - `render(width)` always emits the *current* frame's escape sequence
+ *    (no across-frame cache). Same line-shape trick as pi-tui's `Image`:
+ *    return `rows` lines, the last carrying the cursor-up + image
+ *    transmission so the bitmap occupies all `rows` of vertical space.
+ *  - The `setInterval` ticks `currentFrame` and calls
+ *    `tui.requestRender()` — pi-tui re-runs the diff renderer, our
+ *    `render()` produces a new escape, the terminal swaps the bitmap
+ *    in place by image ID. Same pattern pi-tui's spinner uses for the
+ *    text spinner; typing keeps working because input handling is on
+ *    an independent code path.
+ *  - The interval is `unref()`ed so it doesn't keep the node process
+ *    alive on shutdown.
+ *  - `dispose()` clears the interval. Call it when the sim unloads or
+ *    when a new sim takes over the active-animation slot.
+ *  - In terminals without `kitty` / `iterm2` capability the component
+ *    emits a single line of fallback text instead — but in practice
+ *    the message renderer never instantiates `AnimatedImage` on those
+ *    terminals; it returns the ANSI half-block `Text` path.
+ */
+
+import {
+  type Component,
+  type ImageDimensions,
+  type TUI,
+  encodeKitty,
+  encodeITerm2,
+  getCapabilities,
+  calculateImageRows,
+  getCellDimensions,
+} from "@mariozechner/pi-tui";
+
+export interface AnimatedImageOptions {
+  /** Pre-encoded base64 PNG bytes per frame, in playback order.
+   *  At least 2 frames; otherwise just use a static `Image` instead. */
+  frames: string[];
+  /** Native frame dimensions in pixels — used both for aspect ratio
+   *  during display-cell calculation and for the `c=,r=` parameters
+   *  on the Kitty escape. Uniform across all frames. */
+  dimensions: ImageDimensions;
+  /** Display target in terminal cells. Aspect-preserved height is
+   *  computed from `dimensions`. */
+  maxWidthCells: number;
+  /** Stable Kitty image ID — every frame transmission carries this so
+   *  the terminal swaps the bitmap in place rather than stacking
+   *  copies. Allocate once per AnimatedImage with `allocateImageId()`. */
+  imageId: number;
+  /** Display rate. Internally clamped to ≤ 16 fps to keep escape
+   *  bandwidth reasonable. */
+  fps: number;
+  /** TUI handle. Required so we can call `requestRender()` after each
+   *  frame tick. `setWidget`'s factory form is the canonical way to
+   *  obtain this from inside an extension. */
+  tui: TUI;
+  /** Fallback text printed when `getCapabilities().images` is null
+   *  (e.g. user forced ANSI mid-render). Defaults to "[animation]". */
+  fallbackText?: string;
+}
+
+export class AnimatedImage implements Component {
+  private readonly frames: string[];
+  private readonly dimensions: ImageDimensions;
+  private readonly maxWidthCells: number;
+  private readonly imageId: number;
+  private readonly tui: TUI;
+  private readonly fallbackText: string;
+
+  /** Index into `frames`. Mutates on each tick. */
+  private currentFrame = 0;
+  /** Active animation timer, or null when stopped / disposed. */
+  private intervalId: ReturnType<typeof setInterval> | null = null;
+  /** True after `dispose()` — render() then short-circuits to a single
+   *  empty line so the diff renderer cleanly removes the image. */
+  private disposed = false;
+
+  constructor(opts: AnimatedImageOptions) {
+    if (opts.frames.length < 2) {
+      throw new Error(
+        `AnimatedImage requires at least 2 frames (got ${opts.frames.length}); use pi-tui's Image for a single frame.`,
+      );
+    }
+    this.frames = opts.frames;
+    this.dimensions = opts.dimensions;
+    this.maxWidthCells = opts.maxWidthCells;
+    this.imageId = opts.imageId;
+    this.tui = opts.tui;
+    this.fallbackText = opts.fallbackText ?? "[animation]";
+
+    // Clamp fps to ≤ 16 — each tick re-transmits the full PNG, ~30–50
+    // KB per codex pet frame. 16 fps = ~800 KB/s, plenty of headroom.
+    const fps = Math.min(Math.max(opts.fps, 1), 16);
+    const intervalMs = Math.round(1000 / fps);
+    this.intervalId = setInterval(() => {
+      if (this.disposed) return;
+      this.currentFrame = (this.currentFrame + 1) % this.frames.length;
+      // Pi-tui's render tick will call our `render()` again, which
+      // emits the new frame's escape sequence. The terminal swaps the
+      // bitmap in place by image ID — no flicker, no scrollback churn.
+      this.tui.requestRender();
+    }, intervalMs);
+    // Don't keep the node process alive solely for the animation
+    // timer — let SIGINT and process exit work cleanly.
+    this.intervalId.unref?.();
+  }
+
+  /** Stop the timer. Idempotent. After dispose, `render()` returns an
+   *  empty single line so pi-tui's diff renderer cleanly clears the
+   *  cells the animation was occupying. */
+  dispose(): void {
+    if (this.intervalId !== null) {
+      clearInterval(this.intervalId);
+      this.intervalId = null;
+    }
+    this.disposed = true;
+  }
+
+  /**
+   * Emit the current frame as a Kitty / iTerm2 inline image escape.
+   * Same line-shape contract pi-tui's `Image` uses: `rows` lines, the
+   * last one containing `\x1b[<rows-1>A` followed by the actual image
+   * transmission. The first `rows-1` lines are empty; pi-tui's diff
+   * renderer treats them as occupied vertical space.
+   */
+  render(width: number): string[] {
+    if (this.disposed) return [""];
+    const caps = getCapabilities();
+    if (!caps.images) {
+      // Should never happen in practice — the message renderer only
+      // builds an AnimatedImage when caps.images is truthy. Soft
+      // fallback so the layout doesn't collapse if caps change at
+      // runtime (e.g. user resizes into iTerm2 from Kitty mid-session).
+      return [this.fallbackText];
+    }
+    const maxWidth = Math.min(width - 2, this.maxWidthCells);
+    const rows = calculateImageRows(this.dimensions, maxWidth, getCellDimensions());
+    const base64 = this.frames[this.currentFrame] ?? this.frames[0];
+
+    let sequence: string;
+    if (caps.images === "kitty") {
+      // imageId is required for in-place swap — that's the whole point
+      // of AnimatedImage. encodeKitty handles the chunked transmission
+      // for payloads above the protocol's 4096-byte chunk limit.
+      sequence = encodeKitty(base64, {
+        columns: maxWidth,
+        rows,
+        imageId: this.imageId,
+      });
+    } else {
+      // iTerm2: no image-ID swap, but it draws the new image at the
+      // current cursor; combined with the cursor-up trick below, the
+      // visual effect is "in place" enough for our needs.
+      sequence = encodeITerm2(base64, {
+        width: maxWidth,
+        height: "auto",
+        preserveAspectRatio: true,
+      });
+    }
+
+    const lines: string[] = [];
+    for (let i = 0; i < rows - 1; i++) lines.push("");
+    const moveUp = rows > 1 ? `\x1b[${rows - 1}A` : "";
+    lines.push(moveUp + sequence);
+    return lines;
+  }
+
+  /** No internal cache to clear — `render()` always recomputes from
+   *  `currentFrame`. Provided for `Component` interface conformance and
+   *  in case pi-tui ever calls invalidate() on the parent tree. */
+  invalidate(): void {
+    /* intentionally empty */
+  }
+}

package/src/index.ts

@@ -60,7 +60,16 @@ import type {
   ExtensionContext,
   ExtensionCommandContext,
 } from "@mariozechner/pi-coding-agent";
-import { Text } from "@mariozechner/pi-tui";
+import {
+  Box,
+  Image,
+  Text,
+  allocateImageId,
+  getCapabilities,
+  type ImageTheme,
+  type TUI,
+} from "@mariozechner/pi-tui";
+import { AnimatedImage } from "./animated-image.ts";
 import { Type } from "typebox";
 
 import {
@@ -83,6 +92,7 @@ import {
   downscaleRgbaNearest,
   boxDownscaleRgba,
 } from "./png-to-ansi.ts";
+import { encodeRgbaToPng } from "./png-encode.ts";
 import { decodeWebp } from "./webp-to-rgba.ts";
 import { openRouterComplete, type ChatMessage } from "./openrouter.ts";
 import { buildSimPrompt, type LoadedSim } from "./persona.ts";
@@ -113,6 +123,107 @@ let loadedSim: LoadedSim | null = null;
  */
 let justUnloaded: string | null = null;
 
+// ---------------------------------------------------------------------------
+// Animation state
+//
+// We animate the most recently loaded codex pet's idle row inline in chat,
+// using the same pattern pi-tui's spinner uses: a setInterval ticks a frame
+// counter and calls `tui.requestRender()`, which makes pi-tui re-run the
+// message renderer. Each render returns a new `Image` keyed by a stable
+// Kitty image ID, so the terminal swaps the displayed frame in place.
+//
+// Typing keeps working during animation — input handling and rendering are
+// independent code paths in pi-tui (verified by inspection of `Loader`).
+//
+// Only ONE sim animates at a time. When a new sim is loaded, the previous
+// timer stops and the previous message freezes on its idle frame.
+// ---------------------------------------------------------------------------
+
+/** Captured pi-tui handle. Set by the `simocracy` widget factory the first
+ *  time a sim is loaded; reused for every subsequent animation tick. Lazy
+ *  because the TUI doesn't exist when the extension first imports. */
+let capturedTui: TUI | null = null;
+
+interface ActiveAnimation {
+  /** Identifies which loaded-sim message owns this animation. The
+   *  renderer compares against `details.animationKey` to decide
+   *  whether to mount the live `AnimatedImage` for this message or a
+   *  static idle frame. */
+  key: string;
+  /** Stable Kitty image ID so frame transmissions replace the previous one
+   *  instead of stacking. Allocated once per sim load. */
+  imageId: number;
+  /** Pre-encoded base64 PNGs in playback order. */
+  frames: string[];
+  /** Frame width / height in pixels (uniform across frames). */
+  widthPx: number;
+  heightPx: number;
+  /** Display rate. */
+  fps: number;
+  /**
+   * The live animated component. Created lazily inside the message
+   * renderer the first time it's asked for this `key` — we need a
+   * TUI handle to construct one, and the renderer is the natural
+   * place that has access (via `capturedTui`). Disposed when a new
+   * sim takes over the active-animation slot or when the sim is
+   * unloaded.
+   */
+  component: AnimatedImage | null;
+}
+let currentAnimation: ActiveAnimation | null = null;
+
+/** Default-on; set `SIMOCRACY_ANIMATION=off` to freeze on idle frame 0. */
+const animationEnabled =
+  (process.env.SIMOCRACY_ANIMATION ?? "on").toLowerCase() !== "off";
+
+/**
+ * Width of the inline sprite render in terminal cells. The source PNG
+ * is always transmitted at native resolution (192×208 for codex pets,
+ * 32×32 for pipoya — see `renderSprite`); this number controls only
+ * how many cells the terminal uses to display it. Aspect ratio is
+ * preserved by pi-tui's `calculateImageRows`. Override with the
+ * `SIMOCRACY_SPRITE_WIDTH` env var.
+ *
+ * 10 cells wide gives a compact ≈6-row inline render that sits
+ * comfortably alongside one or two paragraphs of bio text without
+ * dominating the chat. Bump it to 20 or 32 if you want the sprite
+ * to read at a glance.
+ */
+const spriteWidthCells = (() => {
+  const raw = process.env.SIMOCRACY_SPRITE_WIDTH;
+  const parsed = raw ? Number.parseInt(raw, 10) : NaN;
+  return Number.isFinite(parsed) && parsed >= 4 && parsed <= 120 ? parsed : 10;
+})();
+
+function stopCurrentAnimation(): void {
+  if (currentAnimation?.component) {
+    currentAnimation.component.dispose();
+  }
+  currentAnimation = null;
+}
+
+function startAnimationFor(
+  key: string,
+  frames: { pngBase64: string[]; fps: number; widthPx: number; heightPx: number },
+): void {
+  // Always replace any prior animation — only the most recent
+  // loaded-sim message animates.
+  stopCurrentAnimation();
+  if (!animationEnabled || frames.pngBase64.length < 2) return;
+  currentAnimation = {
+    key,
+    imageId: allocateImageId(),
+    frames: frames.pngBase64,
+    widthPx: frames.widthPx,
+    heightPx: frames.heightPx,
+    fps: frames.fps,
+    // Lazily instantiated by the message renderer the first time it
+    // sees this `key` — we don't have a TUI handle here, only inside
+    // the setWidget factory.
+    component: null,
+  };
+}
+
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------
@@ -153,14 +264,75 @@ const NON_PIXEL_ART_TARGET_LONG_EDGE = 40;
  *              box-downscaled to a comparable terminal size.
  *
  *   absent   — treated as legacy 'pipoya' for back-compat.
+ *
+ * Returns BOTH the ANSI half-block render (always — it's our universal
+ * fallback) AND, when possible, a PNG of the same RGBA cell for inline
+ * terminal-graphics protocols (Kitty / iTerm2). The renderer chooses
+ * which to display based on the host terminal's capabilities.
+ *
+ * The PNG is encoded at the *native* resolution we cropped to (32×32
+ * for pipoya, 192×208 for codexPet, the post-downscale size for the
+ * image fallback). Kitty / iTerm2 do their own scaling to the target
+ * cell box, so passing the native pixels gives the terminal the most
+ * information to work with — pixel art scales up crisply with
+ * nearest-neighbour, codex pet thumbnails scale down cleanly.
+ */
+export interface SpriteRender {
+  ansi: string;
+  png?: { data: Buffer; widthPx: number; heightPx: number };
+  /**
+   * Optional animation frames. Each entry in `pngs` is a PNG of one
+   * cell. Set for codex pets (idle row of their atlas); absent for
+   * everything else (pipoya sprites and image fallbacks render as a
+   * single static frame). The renderer plays these in a loop using
+   * the Kitty in-place image-swap protocol when animation is enabled.
+   */
+  frames?: { pngs: Buffer[]; widthPx: number; heightPx: number; fps: number };
+}
+
+/**
+ * Codex pet atlas constants — keep in sync with the simocracy-v2 hatch-pet
+ * skill that produces these sheets. The atlas is 8 cols × 9 rows of
+ * 192×208 cells; the idle animation lives on row 0, frames 0–5 (the same
+ * default tui-pets ships when a pet.json doesn't override it).
  */
-async function renderSpriteAnsi(sim: SimMatch): Promise<string | null> {
+const CODEX_PET_CELL_W = 192;
+const CODEX_PET_CELL_H = 208;
+const CODEX_PET_COLS = 8;
+const CODEX_PET_IDLE_FRAMES = [0, 1, 2, 3, 4, 5];
+const CODEX_PET_IDLE_FPS = 5;
+
+async function renderSprite(sim: SimMatch): Promise<SpriteRender | null> {
   const spriteKind = sim.sim.spriteKind ?? "pipoya";
   const spriteLink = blobLink(sim.sim.sprite?.ref);
   const petSheetLink = blobLink(sim.sim.petSheet?.ref);
   const petSheetMime = sim.sim.petSheet?.mimeType;
   const imageLink = blobLink(sim.sim.image?.ref);
 
+  /** Render an RGBA region to ANSI half-blocks (shared options). */
+  const toAnsi = (data: Buffer, width: number, height: number) =>
+    renderRgbaToAnsi(data, width, height, {
+      cropToContent: true,
+      cropPad: 1,
+      indent: 2,
+      alphaThreshold: 16,
+    });
+
+  /** Bundle ANSI + PNG of the same RGBA region. PNG-encode is a tiny
+   *  cost and is wrapped in try/catch — if encoding ever fails we
+   *  still return the ANSI render (lossless fallback). */
+  const bundle = (data: Buffer, width: number, height: number): SpriteRender => {
+    const ansi = toAnsi(data, width, height);
+    let png: SpriteRender["png"];
+    try {
+      const pngBytes = encodeRgbaToPng(data, width, height);
+      png = { data: pngBytes, widthPx: width, heightPx: height };
+    } catch {
+      png = undefined;
+    }
+    return { ansi, png };
+  };
+
   // Pipoya 4×4 walk sheet — the legacy/default path.
   if (spriteKind !== "codexPet" && spriteLink) {
     try {
@@ -170,19 +342,9 @@ async function renderSpriteAnsi(sim: SimMatch): Promise<string | null> {
       if (width >= FRAME && height >= FRAME) {
         // Sheets are 4×4 of 32×32 frames — row 0 col 0 = front-facing walk1.
         const frame = cropRgba(data, width, height, 0, 0, FRAME, FRAME);
-        return renderRgbaToAnsi(frame, FRAME, FRAME, {
-          cropToContent: true,
-          cropPad: 1,
-          indent: 2,
-          alphaThreshold: 16,
-        });
+        return bundle(frame, FRAME, FRAME);
       }
-      return renderRgbaToAnsi(data, width, height, {
-        cropToContent: true,
-        cropPad: 1,
-        indent: 2,
-        alphaThreshold: 16,
-      });
+      return bundle(data, width, height);
     } catch {
       /* fall through to image fallback */
     }
@@ -193,9 +355,12 @@ async function renderSpriteAnsi(sim: SimMatch): Promise<string | null> {
   // is the idle frame. Both PNG and WebP are valid in the lexicon (the
   // hatch-pet skill emits WebP, the dropzone preserves PNG when the
   // user drops a PNG sheet) so we pick the right decoder by mimeType.
-  // We crop the idle cell first thing and box-downscale to ~32 wide,
-  // so the inline render is similar in height to a pipoya sprite
-  // (~17 lines).
+  // We crop the idle cell first thing. For the ANSI render we
+  // box-downscale to ~32 wide so the inline render is similar in
+  // height to a pipoya sprite (~17 lines). For the inline-graphics
+  // PNG we keep the native 192×208 resolution — Kitty / iTerm2 will
+  // scale it down at display time, which preserves more detail than
+  // pre-downscaling here.
   if (spriteKind === "codexPet" && petSheetLink) {
     try {
       const buf = await fetchBlob(sim.did, petSheetLink);
@@ -208,12 +373,46 @@ async function renderSpriteAnsi(sim: SimMatch): Promise<string | null> {
         const targetW = 32;
         const targetH = Math.round((CELL_H / CELL_W) * targetW); // ~35
         const scaled = boxDownscaleRgba(cell, CELL_W, CELL_H, targetW, targetH);
-        return renderRgbaToAnsi(scaled.data, scaled.width, scaled.height, {
-          cropToContent: true,
-          cropPad: 1,
-          indent: 2,
-          alphaThreshold: 16,
-        });
+        const ansi = toAnsi(scaled.data, scaled.width, scaled.height);
+        let png: SpriteRender["png"];
+        try {
+          const pngBytes = encodeRgbaToPng(cell, CELL_W, CELL_H);
+          png = { data: pngBytes, widthPx: CELL_W, heightPx: CELL_H };
+        } catch {
+          png = undefined;
+        }
+
+        // Idle animation frames — same atlas, different cell offsets.
+        // Encoded eagerly here so the message renderer can flip
+        // between them at ~5 FPS without re-decoding the WebP. Wrapped
+        // in its own try/catch — a frame-encoding failure shouldn't
+        // disable the static render.
+        let frames: SpriteRender["frames"];
+        try {
+          const framePngs: Buffer[] = [];
+          for (const idx of CODEX_PET_IDLE_FRAMES) {
+            const cx = (idx % CODEX_PET_COLS) * CELL_W;
+            const cy = Math.floor(idx / CODEX_PET_COLS) * CELL_H;
+            // Skip frames that fall outside the actual atlas extent.
+            if (cx + CELL_W > width || cy + CELL_H > height) continue;
+            const frameCell = cropRgba(data, width, height, cx, cy, CELL_W, CELL_H);
+            framePngs.push(encodeRgbaToPng(frameCell, CELL_W, CELL_H));
+          }
+          // Single-frame "animations" are pointless — leave `frames`
+          // unset so the renderer takes the static path. Two or more
+          // = a real loop.
+          if (framePngs.length >= 2) {
+            frames = {
+              pngs: framePngs,
+              widthPx: CELL_W,
+              heightPx: CELL_H,
+              fps: CODEX_PET_IDLE_FPS,
+            };
+          }
+        } catch {
+          frames = undefined;
+        }
+        return { ansi, png, frames };
       }
     } catch {
       /* fall through to image fallback */
@@ -252,12 +451,7 @@ async function renderSpriteAnsi(sim: SimMatch): Promise<string | null> {
           Math.max(1, Math.round(native.height * k)),
         );
       }
-      return renderRgbaToAnsi(native.data, native.width, native.height, {
-        cropToContent: true,
-        cropPad: 1,
-        indent: 2,
-        alphaThreshold: 16,
-      });
+      return bundle(native.data, native.width, native.height);
     } catch {
       /* fall through */
     }
@@ -284,10 +478,10 @@ async function loadSimByName(query: string): Promise<{
 
 async function hydrateLoadedSim(match: SimMatch): Promise<LoadedSim> {
   // Fetch agents (constitution), style, sprite ANSI + handle in parallel.
-  const [agents, style, spriteAnsi, handle] = await Promise.all([
+  const [agents, style, sprite, handle] = await Promise.all([
     fetchAgentsForSim(match.uri).catch(() => null) as Promise<AgentsRecord | null>,
     fetchStyleForSim(match.uri).catch(() => null) as Promise<StyleRecord | null>,
-    renderSpriteAnsi(match).catch(() => null),
+    renderSprite(match).catch(() => null),
     resolveHandle(match.did).catch(() => null),
   ]);
 
@@ -297,14 +491,36 @@ async function hydrateLoadedSim(match: SimMatch): Promise<LoadedSim> {
     rkey: match.rkey,
     name: match.sim.name,
     handle,
-    spriteAnsi: spriteAnsi ?? undefined,
+    spriteAnsi: sprite?.ansi,
+    spritePng: sprite?.png
+      ? {
+          base64: sprite.png.data.toString("base64"),
+          widthPx: sprite.png.widthPx,
+          heightPx: sprite.png.heightPx,
+        }
+      : undefined,
+    spriteFrames: sprite?.frames
+      ? {
+          pngBase64: sprite.frames.pngs.map((b) => b.toString("base64")),
+          fps: sprite.frames.fps,
+          widthPx: sprite.frames.widthPx,
+          heightPx: sprite.frames.heightPx,
+        }
+      : undefined,
     shortDescription: agents?.shortDescription,
     description: agents?.description,
     style: style?.description,
   };
 }
 
-function formatSimSummary(
+/**
+ * Build the bio text block that appears alongside (or below) the
+ * sprite in the loaded-sim message: name + handle + AT-URI +
+ * shortDescription. Indented two spaces so it lines up with the ANSI
+ * sprite render. Used both as a standalone block (Image+Text
+ * variant) and as the trailing portion of `formatSimSummary`.
+ */
+function formatSimBio(
   sim: LoadedSim,
   theme?: ExtensionContext["ui"]["theme"],
 ): string {
@@ -313,10 +529,6 @@ function formatSimSummary(
     ? (s: string) => theme.fg("accent", s)
     : (s: string) => s;
   const lines: string[] = [];
-  if (sim.spriteAnsi) {
-    lines.push(sim.spriteAnsi);
-    lines.push("");
-  }
   lines.push(`  🐾 ${accent(sim.name)}${sim.handle ? dim(`  @${sim.handle}`) : ""} loaded—pi is now in character.`);
   lines.push(dim(`  ${sim.uri}`));
   if (sim.shortDescription) {
@@ -326,6 +538,19 @@ function formatSimSummary(
   return lines.join("\n");
 }
 
+function formatSimSummary(
+  sim: LoadedSim,
+  theme?: ExtensionContext["ui"]["theme"],
+): string {
+  const lines: string[] = [];
+  if (sim.spriteAnsi) {
+    lines.push(sim.spriteAnsi);
+    lines.push("");
+  }
+  lines.push(formatSimBio(sim, theme));
+  return lines.join("\n");
+}
+
 // The OpenTUI standalone animated viewer used to live here. It now ships
 // alongside this file as `viewer.ts` for anyone who wants the full-window
 // experience — run it manually with:
@@ -411,11 +636,100 @@ export default async function simocracy(pi: ExtensionAPI) {
 
   // -------------------------------------------------------------------------
   // Custom message renderer — shows the sprite + bio inline in the chat.
+  //
+  // Two render paths, picked per-call:
+  //
+  //   1. Inline graphics (preferred when supported). Terminals that
+  //      advertise the Kitty graphics protocol (Kitty, Ghostty, WezTerm,
+  //      Konsole) or iTerm2's inline-image protocol get a real PNG of
+  //      the sprite via pi-tui's `Image` component, stacked above the
+  //      bio text in a `Box`. Pixels are crisp, scaling is the
+  //      terminal's job.
+  //
+  //   2. ANSI half-blocks (universal fallback). Everything else —
+  //      Apple Terminal, tmux without passthrough, plain SSH, dumb
+  //      pipes — falls back to the existing `▀`/`▄` half-block art.
+  //
+  // Override with `SIMOCRACY_INLINE_GRAPHICS=ansi` to force the
+  // half-block path even when the terminal supports inline graphics
+  // (handy for screenshots, demo recordings, or terminals that
+  // *advertise* support but render glitchily).
   // -------------------------------------------------------------------------
-  pi.registerMessageRenderer<{ body: string }>("simocracy_sim_loaded", (message) => {
+  const inlineGraphicsMode =
+    (process.env.SIMOCRACY_INLINE_GRAPHICS ?? "auto").toLowerCase();
+  pi.registerMessageRenderer<SimLoadedDetails>("simocracy_sim_loaded", (message, _opts, theme) => {
+    const details = (message.details as SimLoadedDetails | undefined) ?? {};
     const body =
-      (message.details as { body?: string } | undefined)?.body ??
-      (typeof message.content === "string" ? message.content : "");
+      details.body ?? (typeof message.content === "string" ? message.content : "");
+
+    // Decide whether to use the inline-graphics path. Auto-detect by
+    // default; honour the env-var override either direction.
+    const caps = getCapabilities();
+    const wantGraphics =
+      inlineGraphicsMode === "auto"
+        ? caps.images !== null
+        : inlineGraphicsMode === "kitty" || inlineGraphicsMode === "iterm2";
+    if (wantGraphics && details.spritePngBase64 && details.bioText) {
+      // Pi-tui's Image needs a fallback colour for terminals that
+      // claim image support but later fail to render — we use the
+      // theme's `dim` so the placeholder text is unobtrusive.
+      const imageTheme: ImageTheme = {
+        fallbackColor: theme.fg("dim", "") ? (s: string) => theme.fg("dim", s) : (s: string) => s,
+      };
+      const box = new Box(0, 0);
+
+      // If this message owns the active animation slot, mount a live
+      // `AnimatedImage` (cycles frames, owns its own setInterval). We
+      // cache the component on `currentAnimation.component` so we
+      // don't spawn a fresh timer on every re-render of the message
+      // (pi-tui calls the renderer again on expand/collapse, theme
+      // change, etc.).
+      //
+      // For every other case — older loaded-sim messages whose key
+      // doesn't match, sims without animation frames, or animation
+      // disabled via env — we mount a static `Image` of the idle
+      // frame, which freezes gracefully.
+      const isActiveAnimation =
+        currentAnimation &&
+        capturedTui !== null &&
+        details.animationKey !== undefined &&
+        currentAnimation.key === details.animationKey;
+      if (isActiveAnimation) {
+        if (!currentAnimation!.component) {
+          currentAnimation!.component = new AnimatedImage({
+            frames: currentAnimation!.frames,
+            dimensions: {
+              widthPx: currentAnimation!.widthPx,
+              heightPx: currentAnimation!.heightPx,
+            },
+            maxWidthCells: spriteWidthCells,
+            imageId: currentAnimation!.imageId,
+            fps: currentAnimation!.fps,
+            tui: capturedTui!,
+          });
+        }
+        box.addChild(currentAnimation!.component);
+      } else {
+        // Source PNG is at full native resolution (192×208 for codex
+        // pets, 32×32 for pipoya). The terminal scales it down to
+        // `spriteWidthCells` cells wide on display — we never
+        // pre-downsample on our side, so quality is preserved.
+        box.addChild(
+          new Image(
+            details.spritePngBase64,
+            "image/png",
+            imageTheme,
+            { maxWidthCells: spriteWidthCells },
+            {
+              widthPx: details.spritePngWidth ?? 0,
+              heightPx: details.spritePngHeight ?? 0,
+            },
+          ),
+        );
+      }
+      box.addChild(new Text(details.bioText, 0, 0));
+      return box;
+    }
     return new Text(body, 0, 0);
   });
 
@@ -488,6 +802,7 @@ export default async function simocracy(pi: ExtensionAPI) {
         const name = loadedSim.name;
         loadedSim = null;
         justUnloaded = name;
+        stopCurrentAnimation();
         ctx.ui.setStatus("simocracy", undefined);
         ctx.ui.setWidget("simocracy", undefined);
         ctx.ui.notify(`Unloaded ${name}. Pi will break character on the next reply.`, "info");
@@ -577,6 +892,7 @@ export default async function simocracy(pi: ExtensionAPI) {
       const name = loadedSim.name;
       loadedSim = null;
       justUnloaded = name;
+      stopCurrentAnimation();
       if (ctx.hasUI) {
         ctx.ui.setStatus("simocracy", undefined);
         ctx.ui.setWidget("simocracy", undefined);
@@ -1053,6 +1369,41 @@ async function tryLoadFromQuery(query: string): Promise<LoadedSim | null> {
   return await hydrateLoadedSim(result.matches[0]);
 }
 
+/**
+ * Shape of `details` on the `simocracy_sim_loaded` custom message.
+ * The renderer reads this to choose between the inline-graphics and
+ * ANSI-half-block render paths; both fields are best-effort — the
+ * renderer falls back to the combined `body` string if anything is
+ * missing.
+ */
+interface SimLoadedDetails {
+  uri?: string;
+  did?: string;
+  rkey?: string;
+  name?: string;
+  /** Combined ANSI sprite + bio, used by the half-block fallback path
+   *  and as the textual log content. */
+  body?: string;
+  /** Bio text only (no sprite), used alongside the Image component
+   *  on the inline-graphics path. */
+  bioText?: string;
+  /** base64-encoded PNG of the sprite cell. Triggers the inline-graphics
+   *  path when set + the terminal advertises image support. */
+  spritePngBase64?: string;
+  /** Native PNG width in pixels (aspect ratio for Image scaling). */
+  spritePngWidth?: number;
+  /** Native PNG height in pixels. */
+  spritePngHeight?: number;
+  /**
+   * Identity tag for the active animation, if any. The message renderer
+   * compares against `currentAnimation.key` to decide whether to swap
+   * in the current animation frame or freeze on the static idle PNG.
+   * Stable across re-renders of the same message; differs across
+   * separate sim loads.
+   */
+  animationKey?: string;
+}
+
 async function postSimToChat(
   pi: ExtensionAPI,
   ctx: ExtensionContext,
@@ -1060,21 +1411,54 @@ async function postSimToChat(
   _reload: boolean,
 ) {
   ctx.ui.setStatus("simocracy", `🐾 ${sim.name}`);
-  const headerLines = [`Simocracy: ${sim.name}${sim.handle ? `  (@${sim.handle})` : ""}`];
-  ctx.ui.setWidget("simocracy", headerLines, { placement: "aboveEditor" });
+  // Use the factory form of setWidget so we can capture pi-tui's TUI
+  // handle. We need it to call `requestRender()` from the animation
+  // setInterval (the message renderer doesn't get a TUI reference).
+  // The factory itself returns a tiny static Text widget — the TUI
+  // capture is the actual purpose.
+  const headerText = `Simocracy: ${sim.name}${sim.handle ? `  (@${sim.handle})` : ""}`;
+  ctx.ui.setWidget(
+    "simocracy",
+    (tui) => {
+      capturedTui = tui;
+      return new Text(headerText, 0, 0);
+    },
+    { placement: "aboveEditor" },
+  );
   const body = formatSimSummary(sim, ctx.ui.theme);
+  const bioText = formatSimBio(sim, ctx.ui.theme);
+  // Unique key per load so the renderer can tell which message owns
+  // the active animation. AT-URI + timestamp protects against
+  // re-loading the same sim twice (each load starts its own loop).
+  const animationKey = `${sim.uri}#${Date.now()}`;
+  const details: SimLoadedDetails = {
+    uri: sim.uri,
+    did: sim.did,
+    rkey: sim.rkey,
+    name: sim.name,
+    body,
+    bioText,
+    animationKey,
+  };
+  if (sim.spritePng) {
+    details.spritePngBase64 = sim.spritePng.base64;
+    details.spritePngWidth = sim.spritePng.widthPx;
+    details.spritePngHeight = sim.spritePng.heightPx;
+  }
   pi.sendMessage({
     customType: "simocracy_sim_loaded",
     content: stripAnsiForLog(body),
     display: true,
-    details: {
-      uri: sim.uri,
-      did: sim.did,
-      rkey: sim.rkey,
-      name: sim.name,
-      body,
-    },
+    details,
   });
+  // Kick off (or replace) the animation loop. Only one loop runs at a
+  // time — the last loaded sim animates, earlier messages freeze on
+  // their idle frame.
+  if (sim.spriteFrames) {
+    startAnimationFor(animationKey, sim.spriteFrames);
+  } else {
+    stopCurrentAnimation();
+  }
 }
 
 /** Strip ANSI escapes for the textual log copy (the renderer uses details.body). */

package/src/persona.ts

@@ -18,6 +18,43 @@ export interface LoadedSim {
   style?: string;
   /** Pre-rendered colored ANSI art of the sim's sprite. */
   spriteAnsi?: string;
+  /**
+   * Pre-encoded PNG of the sim's sprite (idle frame), for inline
+   * terminal graphics protocols (Kitty, iTerm2). Present when sprite
+   * rendering succeeded and the source could be encoded; the renderer
+   * uses this when the host terminal advertises image support and
+   * falls back to `spriteAnsi` otherwise.
+   */
+  spritePng?: {
+    /** base64-encoded PNG bytes (no `data:` prefix). */
+    base64: string;
+    /** Native PNG width in pixels (used for aspect ratio). */
+    widthPx: number;
+    /** Native PNG height in pixels. */
+    heightPx: number;
+  };
+  /**
+   * Pre-encoded PNG frames of the sim's idle animation, for terminals
+   * that support inline graphics. When present **and** the renderer's
+   * animation timer is active for this sim, the message renderer
+   * cycles through these instead of repeating `spritePng` — producing
+   * a gentle in-chat idle loop using the same Kitty-protocol
+   * in-place image swap that pi-tui's spinner uses to animate.
+   *
+   * Currently populated only for `codexPet` sims (their atlas defines
+   * an explicit 6-frame idle row); pipoya and image-fallback sims
+   * stay static.
+   */
+  spriteFrames?: {
+    /** base64-encoded PNG bytes per frame, in display order. */
+    pngBase64: string[];
+    /** Display rate in frames-per-second. */
+    fps: number;
+    /** Native PNG width in pixels (uniform across all frames). */
+    widthPx: number;
+    /** Native PNG height in pixels. */
+    heightPx: number;
+  };
 }
 
 /**

package/src/png-encode.ts

@@ -0,0 +1,29 @@
+/**
+ * Encode an RGBA pixel buffer to a PNG (`Buffer`) for inline terminal
+ * graphics protocols (Kitty, iTerm2). Both protocols accept base64-PNG
+ * payloads — Kitty supports it via `f=100`, iTerm2 via `inline=1`.
+ *
+ * Sized to be tiny: the only operation here is wrapping `pngjs`'s
+ * synchronous packer in a typed helper that matches the
+ * decoder/cropper interfaces in `png-to-ansi.ts`. We never need
+ * streaming or async — the largest image we encode is a 192×208 codex
+ * pet idle cell (~30 KB PNG).
+ */
+
+import { PNG } from "pngjs";
+
+/**
+ * Encode a flat RGBA8 buffer to PNG. `data.length` must equal
+ * `width * height * 4`. Returns a fresh `Buffer` containing the PNG
+ * bytes (signature `89 50 4e 47 …`).
+ */
+export function encodeRgbaToPng(data: Buffer, width: number, height: number): Buffer {
+  if (data.length !== width * height * 4) {
+    throw new Error(
+      `encodeRgbaToPng: expected ${width * height * 4} bytes for ${width}×${height}, got ${data.length}`,
+    );
+  }
+  const png = new PNG({ width, height });
+  data.copy(png.data);
+  return PNG.sync.write(png);
+}