pi-cliffcompaction 0.1.0

package/lib/images.ts

@@ -0,0 +1,268 @@
+/**
+ * Token estimation for image blocks.
+ *
+ * Text estimates at chars/4. Images do not: the block is base64 of a
+ * compressed file, and token cost is set by dimensions. Formula is
+ * Anthropic's 28x28-pixel patches, capped at the high-resolution tier.
+ * Overcounting is deliberate: compact a little early rather than send a
+ * request the provider refuses.
+ */
+
+import {
+  asObject,
+  asString,
+  isArray,
+  isRecord,
+  isString,
+  type JsonObject,
+  type JsonValue,
+} from "./decode.ts";
+
+export const PATCH_PX = 28;
+
+export const MAX_IMAGE_TOKENS = 4_784;
+
+export const DEFAULT_IMAGE_TOKENS = 1_568;
+
+const HEADER_B64_CHARS = 8192;
+
+function readUInt32BE(raw: Uint8Array, offset: number): number {
+  return (
+    ((raw[offset] << 24) | (raw[offset + 1] << 16) | (raw[offset + 2] << 8) | raw[offset + 3]) >>> 0
+  );
+}
+
+function readUInt16BE(raw: Uint8Array, offset: number): number {
+  return (raw[offset] << 8) | raw[offset + 1];
+}
+
+function readUInt16LE(raw: Uint8Array, offset: number): number {
+  return raw[offset] | (raw[offset + 1] << 8);
+}
+
+function startsWith(raw: Uint8Array, bytes: number[], offset = 0): boolean {
+  if (raw.length < offset + bytes.length) {
+    return false;
+  }
+
+  for (let i = 0; i < bytes.length; i++) {
+    if (raw[offset + i] !== bytes[i]) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+function pngDimensions(raw: Uint8Array): [number, number] | null {
+  const sig = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
+
+  if (!startsWith(raw, sig) || raw.length < 24) {
+    return null;
+  }
+
+  const width = readUInt32BE(raw, 16);
+  const height = readUInt32BE(raw, 20);
+
+  return [width, height];
+}
+
+function gifDimensions(raw: Uint8Array): [number, number] | null {
+  const gif87 = [0x47, 0x49, 0x46, 0x38, 0x37, 0x61];
+  const gif89 = [0x47, 0x49, 0x46, 0x38, 0x39, 0x61];
+
+  if ((!startsWith(raw, gif87) && !startsWith(raw, gif89)) || raw.length < 10) {
+    return null;
+  }
+
+  return [readUInt16LE(raw, 6), readUInt16LE(raw, 8)];
+}
+
+function jpegDimensions(raw: Uint8Array): [number, number] | null {
+  if (raw.length < 2 || raw[0] !== 0xff || raw[1] !== 0xd8) {
+    return null;
+  }
+
+  let i = 2;
+  const n = raw.length;
+
+  while (i + 9 < n) {
+    if (raw[i] !== 0xff) {
+      i += 1;
+      continue;
+    }
+
+    const marker = raw[i + 1];
+
+    if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd9)) {
+      i += 2;
+      continue;
+    }
+
+    const segLen = readUInt16BE(raw, i + 2);
+
+    if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) {
+      const height = readUInt16BE(raw, i + 5);
+      const width = readUInt16BE(raw, i + 9 - 2);
+
+      return [width, height];
+    }
+
+    if (segLen < 2) {
+      return null;
+    }
+
+    i += 2 + segLen;
+  }
+
+  return null;
+}
+
+function webpDimensions(raw: Uint8Array): [number, number] | null {
+  const riff = [0x52, 0x49, 0x46, 0x46];
+  const webp = [0x57, 0x45, 0x42, 0x50];
+
+  if (!startsWith(raw, riff) || !startsWith(raw, webp, 8) || raw.length < 30) {
+    return null;
+  }
+
+  const fmt = String.fromCharCode(raw[12], raw[13], raw[14], raw[15]);
+
+  if (fmt === "VP8X") {
+    const w = (raw[24] | (raw[25] << 8) | (raw[26] << 16)) + 1;
+    const h = (raw[27] | (raw[28] << 8) | (raw[29] << 16)) + 1;
+
+    return [w, h];
+  }
+
+  if (fmt === "VP8 ") {
+    return [readUInt16LE(raw, 26) & 0x3fff, readUInt16LE(raw, 28) & 0x3fff];
+  }
+
+  if (fmt === "VP8L" && raw.length >= 25) {
+    const bits = raw[21] | (raw[22] << 8) | (raw[23] << 16) | (raw[24] << 24);
+
+    return [(bits & 0x3fff) + 1, ((bits >> 14) & 0x3fff) + 1];
+  }
+
+  return null;
+}
+
+export function dimensions(raw: Uint8Array): [number, number] | null {
+  const readers = [pngDimensions, jpegDimensions, gifDimensions, webpDimensions];
+
+  for (const reader of readers) {
+    try {
+      const dims = reader(raw);
+
+      if (dims && dims[0] > 0 && dims[1] > 0) {
+        return dims;
+      }
+    } catch {
+      continue;
+    }
+  }
+
+  return null;
+}
+
+function headerBytes(payload: string): Uint8Array | null {
+  const comma = payload.startsWith("data:") ? payload.indexOf(",") : -1;
+  let b64 = comma >= 0 ? payload.slice(comma + 1) : payload;
+  b64 = b64.slice(0, HEADER_B64_CHARS);
+  b64 = b64.slice(0, b64.length - (b64.length % 4));
+
+  if (!b64) {
+    return null;
+  }
+
+  try {
+    return Buffer.from(b64, "base64");
+  } catch {
+    return null;
+  }
+}
+
+function ceilDiv(n: number, d: number): number {
+  return Math.trunc((n + d - 1) / d);
+}
+
+export function tokensForPayload(payload: string): number {
+  if (!payload.startsWith("data:")) {
+    return DEFAULT_IMAGE_TOKENS;
+  }
+
+  const raw = headerBytes(payload);
+  const dims = raw ? dimensions(raw) : null;
+
+  if (dims === null) {
+    return DEFAULT_IMAGE_TOKENS;
+  }
+
+  const patches = ceilDiv(dims[0], PATCH_PX) * ceilDiv(dims[1], PATCH_PX);
+
+  return Math.max(1, Math.min(patches, MAX_IMAGE_TOKENS));
+}
+
+export function imagePayloadFromNode(obj: JsonObject): string | null {
+  const kind = asString(obj.type);
+
+  if (kind === "image") {
+    const source = asObject(obj.source);
+
+    if (source && isString(source.data)) {
+      return source.data;
+    }
+
+    return null;
+  }
+
+  if (kind === "input_image" || kind === "image_url") {
+    const urlField = obj.image_url;
+
+    if (isString(urlField)) {
+      return urlField;
+    }
+
+    const nested = asObject(urlField);
+
+    if (nested && isString(nested.url)) {
+      return nested.url;
+    }
+
+    return null;
+  }
+
+  return null;
+}
+
+export function imagePayloads(obj: JsonValue): string[] {
+  const out: string[] = [];
+  collectPayloads(obj, out);
+
+  return out;
+}
+
+function collectPayloads(obj: JsonValue, out: string[]): void {
+  if (isRecord(obj)) {
+    const payload = imagePayloadFromNode(obj);
+
+    if (payload !== null) {
+      out.push(payload);
+
+      return;
+    }
+
+    for (const key of Object.keys(obj)) {
+      collectPayloads(obj[key], out);
+    }
+
+    return;
+  }
+
+  if (isArray(obj)) {
+    for (const item of obj) {
+      collectPayloads(item, out);
+    }
+  }
+}

package/lib/index.ts

@@ -0,0 +1,62 @@
+/**
+ * CliffCompaction library: mechanical, high-precision context compaction.
+ *
+ * Nguyen, Cho, Chen & Dettmers, arXiv:2609.26779.
+ */
+
+export { makeConfig, replaceConfig, parseConfig, loadConfig, configFromEnv, DEFAULT_CONFIG } from "./config.ts";
+
+export type { Config, ConfigPatch } from "./config.ts";
+
+export { compact, groupTurns } from "./cliff.ts";
+
+export type { CompactResult } from "./cliff.ts";
+
+export { Engine, estimateTokens, billableChars, messageChars, ctxOutgoingBody, summaryFingerprint } from "./engine.ts";
+
+export type { RequestCtx } from "./engine.ts";
+
+export { PrefixStore, makeEntry, entrySize } from "./store.ts";
+
+export type { Entry } from "./store.ts";
+
+export { chainHashes, digestObj, digestBytes } from "./hashing.ts";
+
+export { canonicalJson, dumpsDefault } from "./json.ts";
+
+export {
+  dimensions,
+  tokensForPayload,
+  imagePayloads,
+  MAX_IMAGE_TOKENS,
+  DEFAULT_IMAGE_TOKENS,
+  PATCH_PX,
+} from "./images.ts";
+
+export {
+  detect,
+  anthropicDialect,
+  openaiChatDialect,
+  openaiResponsesDialect,
+  piDialect,
+  SUMMARY_HEADER,
+  PI_COMPACTION_PREFIX,
+  truncate,
+  stripTaskNotifications,
+} from "./dialects/index.ts";
+
+export type { Dialect } from "./dialects/index.ts";
+
+export { compactSession, liveFromEntries } from "./pi-hook.ts";
+
+export type {
+  CompactSessionInput,
+  CompactSessionOutput,
+  SessionMessageRef,
+  HookEntry,
+  CliffDetails,
+} from "./pi-hook.ts";
+
+export type { JsonObject, JsonValue, JsonArray } from "./decode.ts";
+
+export { isString, isRecord, isArray, asString, asObject, asArray } from "./decode.ts";

package/lib/json.ts

@@ -0,0 +1,256 @@
+/**
+ * JSON serializers matching CPython json.dumps used by the reference
+ * implementation: canonical (sorted, compact) for hashing, default
+ * (insertion-order, ", "/": " separators, ensure_ascii=False) for
+ * billable character counts.
+ *
+ * dumpsLen counts without building the string (same length as dumpsDefault).
+ */
+
+import {
+  isArray,
+  isBoolean,
+  isNull,
+  isNumber,
+  isRecord,
+  isString,
+  type JsonObject,
+  type JsonValue,
+} from "./decode.ts";
+
+const KEY_JSON_LEN = new Map();
+
+function emitString(value: string): string {
+  return JSON.stringify(value);
+}
+
+function keyJsonLen(key: string): number {
+  const hit = KEY_JSON_LEN.get(key);
+
+  if (hit !== undefined) {
+    return hit;
+  }
+
+  const n = emitString(key).length;
+  KEY_JSON_LEN.set(key, n);
+
+  return n;
+}
+
+function emitNumber(value: number): string {
+  return JSON.stringify(value);
+}
+
+function pushCanonical(value: JsonValue, parts: { push(chunk: string): void }): void {
+  if (isNull(value)) {
+    parts.push("null");
+
+    return;
+  }
+
+  if (isBoolean(value)) {
+    parts.push(value ? "true" : "false");
+
+    return;
+  }
+
+  if (isNumber(value)) {
+    parts.push(emitNumber(value));
+
+    return;
+  }
+
+  if (isString(value)) {
+    parts.push(emitString(value));
+
+    return;
+  }
+
+  if (isArray(value)) {
+    parts.push("[");
+
+    for (let i = 0; i < value.length; i++) {
+      if (i !== 0) {
+        parts.push(",");
+      }
+
+      pushCanonical(value[i], parts);
+    }
+
+    parts.push("]");
+
+    return;
+  }
+
+  if (!isRecord(value)) {
+    throw new TypeError("canonicalJson: not JSON");
+  }
+
+  const keys = Object.keys(value).sort();
+  parts.push("{");
+
+  for (let i = 0; i < keys.length; i++) {
+    if (i !== 0) {
+      parts.push(",");
+    }
+
+    const key = keys[i];
+    parts.push(emitString(key));
+    parts.push(":");
+    pushCanonical(value[key], parts);
+  }
+
+  parts.push("}");
+}
+
+/** json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False) */
+export function canonicalJson(value: JsonValue): string {
+  const parts: string[] = [];
+  pushCanonical(value, parts);
+
+  return parts.join("");
+}
+
+function pushDefault(value: JsonValue, parts: string[]): void {
+  if (isNull(value)) {
+    parts.push("null");
+
+    return;
+  }
+
+  if (isBoolean(value)) {
+    parts.push(value ? "true" : "false");
+
+    return;
+  }
+
+  if (isNumber(value)) {
+    parts.push(emitNumber(value));
+
+    return;
+  }
+
+  if (isString(value)) {
+    parts.push(emitString(value));
+
+    return;
+  }
+
+  if (isArray(value)) {
+    parts.push("[");
+
+    for (let i = 0; i < value.length; i++) {
+      if (i !== 0) {
+        parts.push(", ");
+      }
+
+      pushDefault(value[i], parts);
+    }
+
+    parts.push("]");
+
+    return;
+  }
+
+  if (!isRecord(value)) {
+    throw new TypeError("dumpsDefault: not JSON");
+  }
+
+  const keys = Object.keys(value);
+  parts.push("{");
+
+  for (let i = 0; i < keys.length; i++) {
+    if (i !== 0) {
+      parts.push(", ");
+    }
+
+    const key = keys[i];
+    parts.push(emitString(key));
+    parts.push(": ");
+    pushDefault(value[key], parts);
+  }
+
+  parts.push("}");
+}
+
+/** json.dumps(obj, ensure_ascii=False) -- default separators. */
+export function dumpsDefault(value: JsonValue): string {
+  const parts: string[] = [];
+  pushDefault(value, parts);
+
+  return parts.join("");
+}
+
+/** Character count of dumpsDefault(value), without allocating the string. */
+export function dumpsCount(value: JsonValue, onRecord?: (obj: JsonObject) => void): number {
+  if (isNull(value)) {
+    return 4;
+  }
+
+  if (isBoolean(value)) {
+    return value ? 4 : 5;
+  }
+
+  if (isNumber(value)) {
+    return emitNumber(value).length;
+  }
+
+  if (isString(value)) {
+    return emitString(value).length;
+  }
+
+  if (isArray(value)) {
+    let n = 2;
+
+    for (let i = 0; i < value.length; i++) {
+      if (i !== 0) {
+        n += 2;
+      }
+
+      n += dumpsCount(value[i], onRecord);
+    }
+
+    return n;
+  }
+
+  if (!isRecord(value)) {
+    throw new TypeError("dumpsDefault: not JSON");
+  }
+
+  if (onRecord) {
+    onRecord(value);
+  }
+
+  const keys = Object.keys(value);
+  let n = 2;
+
+  for (let i = 0; i < keys.length; i++) {
+    if (i !== 0) {
+      n += 2;
+    }
+
+    n += keyJsonLen(keys[i]) + 2 + dumpsCount(value[keys[i]], onRecord);
+  }
+
+  return n;
+}
+
+export function dumpsLen(value: JsonValue): number {
+  try {
+    return dumpsCount(value);
+  } catch {
+    return 0;
+  }
+}
+
+export function objectWithoutKey(obj: JsonObject, key: string): JsonObject {
+  const out: JsonObject = {};
+
+  for (const k of Object.keys(obj)) {
+    if (k !== key) {
+      out[k] = obj[k];
+    }
+  }
+
+  return out;
+}