pi-sprite 1.0.0

package/skills/pi-sprite-authoring/scripts/openai_sprite_image.py

@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+"""Generate pi-sprite pet images with OpenAI GPT Image.
+
+The script intentionally stays small: one prompt in, optional reference images,
+one generated sprite candidate out. Use --dry-run to validate prompt/reference
+wiring and write metadata without requiring OPENAI_API_KEY or the openai package.
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import json
+import mimetypes
+import os
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+SUPPORTED_REFERENCE_SUFFIXES = {".png", ".webp", ".jpg", ".jpeg"}
+DEFAULT_MODEL = "gpt-image-2"
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Generate a pi-sprite image with OpenAI GPT Image.")
+    prompt_group = parser.add_mutually_exclusive_group(required=True)
+    prompt_group.add_argument("--prompt", help="Prompt text to send to OpenAI")
+    prompt_group.add_argument("--prompt-file", type=Path, help="Path to a UTF-8 prompt file")
+    parser.add_argument("--reference-image", type=Path, action="append", default=[], help="Local reference image path; repeatable")
+    parser.add_argument(
+        "--reference-role",
+        action="append",
+        default=[],
+        help="Role for a reference image, e.g. character_reference or style_reference. Repeat in --reference-image order.",
+    )
+    parser.add_argument(
+        "--reference-instruction",
+        action="append",
+        default=[],
+        help="Instruction for a reference image; repeat in --reference-image order. One instruction may be reused for all references.",
+    )
+    parser.add_argument("--output-dir", type=Path, default=Path("generated/pi-sprite"), help="Output directory")
+    parser.add_argument("--prefix", default="sprite", help="Output filename prefix")
+    parser.add_argument("--model", default=os.environ.get("OPENAI_IMAGE_MODEL", DEFAULT_MODEL), help="OpenAI image model")
+    parser.add_argument("--size", default="1024x1024", help="OpenAI image size")
+    parser.add_argument("--quality", default="low", help="OpenAI image quality")
+    parser.add_argument("--output-format", choices=["png", "jpeg", "webp"], default="png", help="Output image format")
+    parser.add_argument(
+        "--background",
+        choices=["transparent", "opaque", "auto"],
+        default="auto",
+        help="OpenAI background setting. Default: auto. Use transparent only with models that support it.",
+    )
+    parser.add_argument("--dry-run", action="store_true", help="Write prompt/metadata without calling OpenAI")
+    return parser.parse_args()
+
+
+def load_prompt(args: argparse.Namespace) -> str:
+    if args.prompt is not None:
+        prompt = args.prompt.strip()
+    else:
+        prompt = args.prompt_file.read_text(encoding="utf-8").strip()
+    if not prompt:
+        raise ValueError("prompt cannot be empty")
+    return prompt
+
+
+def values_for_references(values: list[str], paths: list[Path], *, default: str, flag: str) -> list[str]:
+    if not paths:
+        return []
+    if not values:
+        return [default for _ in paths]
+    if len(values) == 1:
+        return [values[0] for _ in paths]
+    if len(values) != len(paths):
+        raise ValueError(f"{flag} must be supplied once or once per --reference-image")
+    return values
+
+
+def validate_references(paths: list[Path], instructions: list[str], roles: list[str]) -> list[dict[str, str]]:
+    resolved_instructions = values_for_references(instructions, paths, default="", flag="--reference-instruction")
+    resolved_roles = values_for_references(roles, paths, default="reference", flag="--reference-role")
+    if paths and any(not instruction.strip() for instruction in resolved_instructions):
+        raise ValueError("--reference-instruction is required when --reference-image is used")
+    references: list[dict[str, str]] = []
+    for index, path in enumerate(paths, start=1):
+        if path.suffix.lower() not in SUPPORTED_REFERENCE_SUFFIXES:
+            raise ValueError(f"unsupported reference image type: {path}")
+        if not path.exists():
+            raise ValueError(f"reference image missing: {path}")
+        if not path.is_file():
+            raise ValueError(f"reference image is not a file: {path}")
+        if path.stat().st_size <= 0:
+            raise ValueError(f"reference image is empty: {path}")
+        references.append(
+            {
+                "id": f"reference-{index}",
+                "path": str(path),
+                "role": resolved_roles[index - 1].strip() or "reference",
+                "instruction": resolved_instructions[index - 1].strip(),
+            }
+        )
+    return references
+
+
+def openai_image_files(paths: list[Path]) -> list[tuple[str, bytes, str]]:
+    files: list[tuple[str, bytes, str]] = []
+    for path in paths:
+        mime_type = mimetypes.guess_type(path.name)[0] or "image/png"
+        if mime_type == "image/jpg":
+            mime_type = "image/jpeg"
+        files.append((path.name, path.read_bytes(), mime_type))
+    return files
+
+
+def image_data_to_b64(item: Any) -> str | None:
+    if hasattr(item, "b64_json") and item.b64_json:
+        return item.b64_json
+    if isinstance(item, dict):
+        return item.get("b64_json") or item.get("result")
+    return None
+
+
+def prompt_with_reference_bindings(prompt: str, references: list[dict[str, str]]) -> str:
+    if not references:
+        return prompt
+    lines = [prompt, "", "REFERENCE IMAGE BINDINGS:"]
+    for index, reference in enumerate(references, start=1):
+        lines.append(f"{index}. {reference['role']}: {reference['instruction']}")
+    lines.append("")
+    lines.append("Use these bindings when interpreting the attached images. Preserve character_reference identity exactly; use style_reference, scale_reference, plush_reference, modern_style_reference, and other non-character references only for their stated instruction.")
+    return "\n".join(lines)
+
+
+def generate_image(args: argparse.Namespace, prompt: str, references: list[dict[str, str]]) -> tuple[bytes, str | None]:
+    if not os.environ.get("OPENAI_API_KEY"):
+        raise RuntimeError("OPENAI_API_KEY is required unless --dry-run is set")
+    try:
+        from openai import OpenAI
+    except ImportError as exc:
+        raise RuntimeError("Python package 'openai' is required. Run with: uv run --with openai python ...") from exc
+
+    client = OpenAI()
+    kwargs: dict[str, Any] = {
+        "model": args.model,
+        "prompt": prompt_with_reference_bindings(prompt, references),
+        "size": args.size,
+        "quality": args.quality,
+        "output_format": args.output_format,
+        "background": args.background,
+    }
+    if args.reference_image:
+        image_files = openai_image_files(args.reference_image)
+        result = client.images.edit(image=image_files if len(image_files) > 1 else image_files[0], **kwargs)
+        method = "edit"
+    else:
+        result = client.images.generate(**kwargs)
+        method = "generate"
+
+    data = getattr(result, "data", None) or []
+    if not data:
+        raise RuntimeError("OpenAI response did not include image data")
+    b64 = image_data_to_b64(data[0])
+    if not b64:
+        raise RuntimeError("OpenAI response did not include b64 image data")
+    return base64.b64decode(b64), method
+
+
+def write_outputs(args: argparse.Namespace, prompt: str, references: list[dict[str, str]]) -> dict[str, Any]:
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+    base = args.output_dir / f"{args.prefix}-{stamp}"
+    method = "edit" if args.reference_image else "generate"
+    image_path: Path | None = None
+    if not args.dry_run:
+        image_bytes, method = generate_image(args, prompt, references)
+        image_path = base.with_suffix(f".{args.output_format}")
+        image_path.write_bytes(image_bytes)
+
+    effective_prompt = prompt_with_reference_bindings(prompt, references)
+    prompt_path = base.with_suffix(".prompt.txt")
+    prompt_path.write_text(effective_prompt + "\n", encoding="utf-8")
+    metadata = {
+        "dry_run": args.dry_run,
+        "method": method,
+        "model": args.model,
+        "size": args.size,
+        "quality": args.quality,
+        "output_format": args.output_format,
+        "background": args.background,
+        "prompt_path": str(prompt_path),
+        "reference_bindings_in_prompt": bool(references),
+        "image_path": str(image_path) if image_path else None,
+        "references": references,
+    }
+    metadata_path = base.with_suffix(".metadata.json")
+    metadata["metadata_path"] = str(metadata_path)
+    metadata_path.write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return metadata
+
+
+def main() -> int:
+    try:
+        args = parse_args()
+        prompt = load_prompt(args)
+        references = validate_references(args.reference_image, args.reference_instruction, args.reference_role)
+        metadata = write_outputs(args, prompt, references)
+    except Exception as exc:
+        print(f"ERROR: {exc}", file=sys.stderr)
+        return 2
+
+    print(f"dry_run: {metadata['dry_run']}")
+    print(f"method: {metadata['method']}")
+    print(f"prompt: {metadata['prompt_path']}")
+    print(f"metadata: {metadata['metadata_path']}")
+    if metadata["image_path"]:
+        print(f"image: {metadata['image_path']}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/skills/pi-sprite-authoring/scripts/remove_sprite_background.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+"""Remove edge-connected flat/checker backgrounds from generated pi-sprite images.
+
+This is a local post-processing helper. It does not call network APIs. It uses
+Pillow to make edge-connected background pixels transparent, then optionally
+normalizes the sprite onto a square canvas for pi-sprite imports.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import sys
+from collections import deque
+from pathlib import Path
+from typing import Any
+
+
+def parse_color(value: str) -> tuple[int, int, int]:
+    text = value.strip().removeprefix("#")
+    if len(text) != 6:
+        raise argparse.ArgumentTypeError("colors must be 6-digit hex, e.g. #ffffff")
+    try:
+        return (int(text[0:2], 16), int(text[2:4], 16), int(text[4:6], 16))
+    except ValueError as exc:
+        raise argparse.ArgumentTypeError("colors must be 6-digit hex, e.g. #ffffff") from exc
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Remove an edge-connected background from a generated pi-sprite image.")
+    parser.add_argument("--input", type=Path, required=True, help="Input PNG/JPEG/WebP image")
+    parser.add_argument("--output", type=Path, required=True, help="Output PNG path")
+    parser.add_argument("--metadata", type=Path, help="Optional metadata JSON path")
+    parser.add_argument("--target-size", type=int, default=128, help="Final square canvas size in pixels; use 0 to preserve size")
+    parser.add_argument("--padding", type=int, default=10, help="Padding around cropped sprite when target-size is set")
+    parser.add_argument(
+        "--threshold",
+        type=float,
+        default=38.0,
+        help="RGB distance threshold for background matching. Raise for checker/near-white backgrounds.",
+    )
+    parser.add_argument(
+        "--background-color",
+        type=parse_color,
+        action="append",
+        default=[],
+        help="Background color to remove, as #RRGGBB. Repeatable. Defaults to sampled edge colors.",
+    )
+    parser.add_argument("--dry-run", action="store_true", help="Write metadata only; do not write the output image")
+    return parser.parse_args()
+
+
+def color_distance(a: tuple[int, int, int], b: tuple[int, int, int]) -> float:
+    return math.sqrt(sum((a[index] - b[index]) ** 2 for index in range(3)))
+
+
+def close_to_any(color: tuple[int, int, int], samples: list[tuple[int, int, int]], threshold: float) -> bool:
+    return any(color_distance(color, sample) <= threshold for sample in samples)
+
+
+def edge_positions(width: int, height: int) -> list[tuple[int, int]]:
+    positions: list[tuple[int, int]] = []
+    for x in range(width):
+        positions.append((x, 0))
+        positions.append((x, height - 1))
+    for y in range(1, height - 1):
+        positions.append((0, y))
+        positions.append((width - 1, y))
+    return positions
+
+
+def sampled_edge_colors(image: Any, explicit: list[tuple[int, int, int]]) -> list[tuple[int, int, int]]:
+    if explicit:
+        return explicit
+    width, height = image.size
+    candidates = [
+        (0, 0),
+        (width - 1, 0),
+        (0, height - 1),
+        (width - 1, height - 1),
+        (width // 2, 0),
+        (width // 2, height - 1),
+        (0, height // 2),
+        (width - 1, height // 2),
+    ]
+    seen: set[tuple[int, int, int]] = set()
+    samples: list[tuple[int, int, int]] = []
+    pixels = image.load()
+    for pos in candidates:
+        pixel = pixels[pos[0], pos[1]]
+        color = (pixel[0], pixel[1], pixel[2])
+        if color not in seen:
+            seen.add(color)
+            samples.append(color)
+    return samples
+
+
+def transparent_background_mask(image: Any, samples: list[tuple[int, int, int]], threshold: float) -> set[tuple[int, int]]:
+    width, height = image.size
+    pixels = image.load()
+    visited: set[tuple[int, int]] = set()
+    queue: deque[tuple[int, int]] = deque()
+
+    for position in edge_positions(width, height):
+        pixel = pixels[position[0], position[1]]
+        if pixel[3] == 0 or close_to_any((pixel[0], pixel[1], pixel[2]), samples, threshold):
+            visited.add(position)
+            queue.append(position)
+
+    while queue:
+        x, y = queue.popleft()
+        for nx, ny in ((x - 1, y), (x + 1, y), (x, y - 1), (x, y + 1)):
+            if nx < 0 or ny < 0 or nx >= width or ny >= height or (nx, ny) in visited:
+                continue
+            pixel = pixels[nx, ny]
+            if pixel[3] == 0 or close_to_any((pixel[0], pixel[1], pixel[2]), samples, threshold):
+                visited.add((nx, ny))
+                queue.append((nx, ny))
+    return visited
+
+
+def normalize_canvas(image: Any, target_size: int, padding: int) -> tuple[Any, tuple[int, int, int, int] | None]:
+    if target_size <= 0:
+        return image, image.getbbox()
+    bbox = image.getbbox()
+    if bbox is None:
+        return image.resize((target_size, target_size)), None
+
+    from PIL import Image
+
+    cropped = image.crop(bbox)
+    max_sprite_size = max(1, target_size - 2 * padding)
+    scale = min(max_sprite_size / cropped.width, max_sprite_size / cropped.height, 1.0)
+    resized = cropped.resize((max(1, round(cropped.width * scale)), max(1, round(cropped.height * scale))), Image.Resampling.LANCZOS)
+    canvas = Image.new("RGBA", (target_size, target_size), (0, 0, 0, 0))
+    left = (target_size - resized.width) // 2
+    top = target_size - padding - resized.height
+    canvas.alpha_composite(resized, (left, max(0, top)))
+    return canvas, bbox
+
+
+def process(args: argparse.Namespace) -> dict[str, Any]:
+    if not args.input.exists():
+        raise ValueError(f"input image missing: {args.input}")
+    if args.target_size < 0:
+        raise ValueError("--target-size must be >= 0")
+    if args.padding < 0:
+        raise ValueError("--padding must be >= 0")
+
+    from PIL import Image
+
+    original = Image.open(args.input)
+    image = original.convert("RGBA")
+    width, height = image.size
+    samples = sampled_edge_colors(image, args.background_color)
+    background_pixels = transparent_background_mask(image, samples, args.threshold)
+
+    pixels = image.load()
+    for x, y in background_pixels:
+        r, g, b, _ = pixels[x, y]
+        pixels[x, y] = (r, g, b, 0)
+
+    normalized, bbox = normalize_canvas(image, args.target_size, args.padding)
+    if not args.dry_run:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        normalized.save(args.output, format="PNG")
+
+    metadata = {
+        "input": str(args.input),
+        "output": str(args.output),
+        "dry_run": args.dry_run,
+        "original_mode": original.mode,
+        "original_size": [width, height],
+        "target_size": args.target_size,
+        "padding": args.padding,
+        "threshold": args.threshold,
+        "background_samples": [f"#{r:02x}{g:02x}{b:02x}" for r, g, b in samples],
+        "removed_edge_connected_pixels": len(background_pixels),
+        "nontransparent_bbox": list(bbox) if bbox else None,
+        "output_size": list(normalized.size),
+        "has_alpha": True,
+    }
+    if args.metadata:
+        args.metadata.parent.mkdir(parents=True, exist_ok=True)
+        args.metadata.write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return metadata
+
+
+def main() -> int:
+    try:
+        args = parse_args()
+        metadata = process(args)
+    except Exception as exc:
+        print(f"ERROR: {exc}", file=sys.stderr)
+        return 2
+
+    print(f"input: {metadata['input']}")
+    print(f"output: {metadata['output']}")
+    print(f"dry_run: {metadata['dry_run']}")
+    print(f"removed_edge_connected_pixels: {metadata['removed_edge_connected_pixels']}")
+    print(f"output_size: {metadata['output_size']}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/src/agent/session-entries.ts

@@ -0,0 +1,28 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+
+export const BTW_ENTRY = "pi-sprite:btw-entry";
+export const BTW_RESET = "pi-sprite:btw-reset";
+export const RECAP_ENTRY = "pi-sprite:recap";
+
+const HIDDEN_CONTEXT_CUSTOM_TYPES = new Set([BTW_ENTRY, BTW_RESET, RECAP_ENTRY]);
+
+export function customTypeOf(entry: unknown): string | undefined {
+	if (!entry || typeof entry !== "object") return undefined;
+	const typed = entry as { customType?: unknown };
+	return typeof typed.customType === "string" ? typed.customType : undefined;
+}
+
+export function isPiSpriteHiddenContextEntry(entry: unknown): boolean {
+	const customType = customTypeOf(entry);
+	return Boolean(customType && HIDDEN_CONTEXT_CUSTOM_TYPES.has(customType));
+}
+
+export function filterPiSpriteHiddenContextEntries<T>(entries: T[]): T[] {
+	return entries.filter((entry) => !isPiSpriteHiddenContextEntry(entry));
+}
+
+export function registerPiSpriteContextFilter(pi: ExtensionAPI): void {
+	(
+		pi as { on: (event: string, handler: (event: { messages: unknown[] }) => Promise<{ messages: unknown[] }>) => void }
+	).on("context", async (event) => ({ messages: filterPiSpriteHiddenContextEntries(event.messages) }));
+}

package/src/agent/side-completion.ts

@@ -0,0 +1,94 @@
+import type { Message, TextContent } from "@earendil-works/pi-ai/compat";
+import type { ExtensionCommandContext, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import type { SideCompletionRequest, SideCompletionResult } from "./side-session-types.ts";
+
+type CompletionContext = ExtensionCommandContext | ExtensionContext;
+
+type DirectCompletionResult = { ok: true; text: string } | { ok: false; message: string };
+
+export async function withTimeout<T>(promise: Promise<T>, ms: number): Promise<T | undefined> {
+	return await new Promise<T | undefined>((resolve) => {
+		const timer = setTimeout(() => resolve(undefined), ms);
+		promise.then(
+			(value) => {
+				clearTimeout(timer);
+				resolve(value);
+			},
+			() => {
+				clearTimeout(timer);
+				resolve(undefined);
+			},
+		);
+	});
+}
+
+function cappedModel<T extends { maxTokens?: number }>(model: T, maxTokens: number): T {
+	return {
+		...model,
+		maxTokens: Math.min(model.maxTokens ?? maxTokens, maxTokens),
+	};
+}
+
+export async function completeWithApiKeyText(
+	ctx: CompletionContext,
+	prompt: string,
+	options: { maxTokens: number; timeoutMs?: number; systemPrompt?: string },
+): Promise<DirectCompletionResult> {
+	if (!ctx.model) return { ok: false, message: "No active model selected." };
+	const auth = await ctx.modelRegistry.getApiKeyAndHeaders(ctx.model);
+	if (!auth.ok || !auth.apiKey) return { ok: false, message: auth.ok ? "No API key available." : auth.error };
+	const request = async () => {
+		const messages: Message[] = [{ role: "user", content: [{ type: "text", text: prompt }], timestamp: Date.now() }];
+		const { complete } = await import("@earendil-works/pi-ai/compat");
+		const response = await complete(
+			cappedModel(ctx.model!, options.maxTokens),
+			{ systemPrompt: options.systemPrompt, messages },
+			{ apiKey: auth.apiKey, headers: auth.headers, maxTokens: options.maxTokens },
+		);
+		return response.content
+			.filter((part): part is TextContent => part.type === "text")
+			.map((part) => part.text)
+			.join("\n")
+			.trim();
+	};
+	const text = options.timeoutMs ? await withTimeout(request(), options.timeoutMs) : await request();
+	if (!text)
+		return {
+			ok: false,
+			message: options.timeoutMs
+				? "Direct API-key fallback timed out or returned no text."
+				: "Direct API-key fallback returned no text.",
+		};
+	return { ok: true, text };
+}
+
+export async function completeTextWithSideSessionFallback(
+	ctx: CompletionContext,
+	request: SideCompletionRequest,
+	adapters: {
+		sideSession: (ctx: CompletionContext, request: SideCompletionRequest) => Promise<SideCompletionResult>;
+		direct: (
+			ctx: CompletionContext,
+			prompt: string,
+			maxTokens: number,
+		) => Promise<string | { ok: false; message: string } | undefined>;
+	},
+): Promise<
+	| { ok: true; text: string; source: "side-session" | "api-key-fallback" }
+	| { ok: false; sideResult: Extract<SideCompletionResult, { ok: false }>; directMessage: string; message: string }
+> {
+	const sideResult = await adapters.sideSession(ctx, request);
+	if (sideResult.ok) return { ok: true, text: sideResult.text, source: "side-session" };
+	const directResult = await adapters.direct(ctx, request.prompt, request.maxTokens ?? 1200);
+	if (typeof directResult === "string" && directResult) {
+		return { ok: true, text: directResult, source: "api-key-fallback" };
+	}
+	const directMessage =
+		typeof directResult === "object" ? directResult.message : "Direct API-key fallback returned no text.";
+	return {
+		ok: false,
+		sideResult,
+		directMessage,
+		message: "Side session and direct API-key fallback returned no text.",
+	};
+}

package/src/agent/side-session-text.ts

@@ -0,0 +1,20 @@
+export function extractAssistantText(messages: unknown[]): string {
+	for (const message of [...messages].reverse()) {
+		const role = (message as { role?: string })?.role;
+		if (role !== "assistant") continue;
+		const content = (message as { content?: unknown })?.content;
+		if (typeof content === "string") return content.trim();
+		if (!Array.isArray(content)) continue;
+		const text = content
+			.map((part) =>
+				part && typeof part === "object" && (part as { type?: string }).type === "text"
+					? ((part as { text?: string }).text ?? "")
+					: "",
+			)
+			.filter(Boolean)
+			.join("\n")
+			.trim();
+		if (text) return text;
+	}
+	return "";
+}

package/src/agent/side-session-types.ts

@@ -0,0 +1,12 @@
+export type SideCompletionRequest = {
+	prompt: string;
+	systemPrompt?: string;
+	maxTokens?: number;
+	timeoutMs?: number;
+};
+
+export type SideCompletionFailureReason = "no-model" | "timeout" | "empty" | "error";
+
+export type SideCompletionResult =
+	| { ok: true; text: string }
+	| { ok: false; reason: SideCompletionFailureReason; message: string };