promptopskit 0.7.1 → 0.8.0

package/dist/providers/openai.cjs

@@ -35,16 +35,17 @@ __export(openai_exports, {
 module.exports = __toCommonJS(openai_exports);
 
 // src/renderer/interpolate.ts
-var VARIABLE_RE = /\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}/g;
+var VARIABLE_RE = /\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)(?:\s*\|\s*(compress|toon|compact|code))?\s*\}\}/g;
 var ESCAPED_OPEN = /\\\{\\\{/g;
 var ESCAPE_PLACEHOLDER = "\0ESCAPED_OPEN\0";
 function interpolate(template, variables, options = {}) {
   const { strict = false } = options;
   const optionalVariables = new Set(options.optionalVariables ?? []);
   let result = template.replace(ESCAPED_OPEN, ESCAPE_PLACEHOLDER);
-  result = result.replace(VARIABLE_RE, (match, name) => {
+  result = result.replace(VARIABLE_RE, (match, name, modifier) => {
     if (name in variables) {
-      return variables[name];
+      const value = variables[name];
+      return options.transformVariable ? options.transformVariable({ name, value, modifier, match }) : value;
     }
     if (strict && !optionalVariables.has(name)) {
       throw new Error(`Missing required variable: "${name}"`);
@@ -72,12 +73,56 @@ function normalizeContextInput(input) {
     warnings: input.warnings,
     max_size: input.max_size,
     trim: input.trim,
+    compression: normalizeContextInputCompression(input.compression),
     allow_regex: normalizeContextRegex(input.allow_regex),
     deny_regex: normalizeContextRegex(input.deny_regex),
     non_empty: normalizeBuiltInValidator(input.non_empty),
     reject_secrets: normalizeBuiltInValidator(input.reject_secrets)
   };
 }
+function normalizeContextInputCompression(value) {
+  if (value === void 0) {
+    return void 0;
+  }
+  if (value === "heuristic") {
+    return { heuristic: { enabled: true } };
+  }
+  if (value === "code") {
+    return { code: { enabled: true } };
+  }
+  if (value.heuristic === void 0) {
+    return normalizeCodeCompression(value.code);
+  }
+  const normalized = {
+    heuristic: {
+      enabled: value.heuristic.enabled ?? true,
+      min_tokens: value.heuristic.min_tokens,
+      max_sentences: value.heuristic.max_sentences,
+      target_reduction: value.heuristic.target_reduction,
+      query: value.heuristic.query,
+      query_variable: value.heuristic.query_variable,
+      json_to_toon: value.heuristic.json_to_toon
+    }
+  };
+  const code = normalizeCodeCompression(value.code);
+  if (code?.code) {
+    normalized.code = code.code;
+  }
+  return normalized;
+}
+function normalizeCodeCompression(value) {
+  if (value === void 0) {
+    return void 0;
+  }
+  return {
+    code: {
+      enabled: value.enabled ?? true,
+      remove_comments: value.remove_comments,
+      trim_indentation: value.trim_indentation,
+      collapse_blank_lines: value.collapse_blank_lines
+    }
+  };
+}
 function normalizeContextRegex(value) {
   if (value === void 0) {
     return void 0;
@@ -331,27 +376,703 @@ function measureContextValueSize(value) {
   return textEncoder.encode(value).length;
 }
 
+// src/toon-encoding.ts
+var DEFAULT_INDENT = 2;
+var DEFAULT_DELIMITER = ",";
+function tryJsonToToon(text, options = {}) {
+  const trimmed = text.trim();
+  if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) {
+    return void 0;
+  }
+  let parsed;
+  try {
+    parsed = JSON.parse(trimmed);
+  } catch {
+    return void 0;
+  }
+  const value = normalizeJsonValue(parsed);
+  if (value === void 0) {
+    return void 0;
+  }
+  return {
+    output: encodeToon(value, options),
+    value
+  };
+}
+function encodeToon(value, options = {}) {
+  const resolved = {
+    indent: options.indent ?? DEFAULT_INDENT,
+    delimiter: options.delimiter ?? DEFAULT_DELIMITER
+  };
+  return encodeValue(value, void 0, 0, resolved).join("\n");
+}
+function normalizeJsonValue(value) {
+  if (value === null || typeof value === "string" || typeof value === "boolean") {
+    return value;
+  }
+  if (typeof value === "number") {
+    return Number.isFinite(value) ? value : null;
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => normalizeJsonValue(item) ?? null);
+  }
+  if (isPlainObject(value)) {
+    const normalized = {};
+    for (const [key, child] of Object.entries(value)) {
+      normalized[key] = normalizeJsonValue(child) ?? null;
+    }
+    return normalized;
+  }
+  return void 0;
+}
+function encodeValue(value, key, depth, options) {
+  if (isPrimitive(value)) {
+    return [line(depth, key === void 0 ? encodePrimitive(value, options.delimiter) : `${encodeKey(key)}: ${encodePrimitive(value, options.delimiter)}`, options.indent)];
+  }
+  if (Array.isArray(value)) {
+    return encodeArray(value, key, depth, options);
+  }
+  return encodeObject(value, key, depth, options);
+}
+function encodeObject(value, key, depth, options) {
+  const entries = Object.entries(value);
+  const lines = [];
+  if (key !== void 0) {
+    lines.push(line(depth, `${encodeKey(key)}:`, options.indent));
+  }
+  const childDepth = key === void 0 ? depth : depth + 1;
+  for (const [childKey, childValue] of entries) {
+    lines.push(...encodeValue(childValue, childKey, childDepth, options));
+  }
+  return lines;
+}
+function encodeArray(value, key, depth, options) {
+  const prefix = key === void 0 ? "" : encodeKey(key);
+  if (value.length === 0) {
+    return [line(depth, key === void 0 ? "[]" : `${prefix}: []`, options.indent)];
+  }
+  if (value.every(isPrimitive)) {
+    const joined = value.map((item) => encodePrimitive(item, options.delimiter)).join(options.delimiter);
+    return [line(depth, `${prefix}[${value.length}]: ${joined}`, options.indent)];
+  }
+  const tabularFields = getTabularFields(value);
+  if (tabularFields) {
+    const lines2 = [
+      line(depth, `${prefix}[${value.length}]{${tabularFields.map(encodeKey).join(options.delimiter)}}:`, options.indent)
+    ];
+    for (const row of value) {
+      lines2.push(line(depth + 1, tabularFields.map((field) => encodePrimitive(row[field], options.delimiter)).join(options.delimiter), options.indent));
+    }
+    return lines2;
+  }
+  const lines = [line(depth, `${prefix}[${value.length}]:`, options.indent)];
+  for (const item of value) {
+    lines.push(...encodeListItem(item, depth + 1, options));
+  }
+  return lines;
+}
+function encodeListItem(value, depth, options) {
+  if (isPrimitive(value)) {
+    return [line(depth, `- ${encodePrimitive(value, options.delimiter)}`, options.indent)];
+  }
+  if (Array.isArray(value)) {
+    const encoded = encodeArray(value, void 0, depth, options);
+    return encoded.map((item, index) => {
+      if (index === 0) {
+        return line(depth, `- ${item.trimStart()}`, options.indent);
+      }
+      return item;
+    });
+  }
+  const entries = Object.entries(value);
+  if (entries.length === 0) {
+    return [line(depth, "-", options.indent)];
+  }
+  const [firstKey, firstValue] = entries[0];
+  const rest = Object.fromEntries(entries.slice(1));
+  const firstLines = encodeValue(firstValue, firstKey, depth, options);
+  const lines = [line(depth, `- ${firstLines[0].trimStart()}`, options.indent)];
+  for (const continuation of firstLines.slice(1)) {
+    lines.push(continuation);
+  }
+  if (Object.keys(rest).length > 0) {
+    lines.push(...encodeObject(rest, void 0, depth + 1, options));
+  }
+  return lines;
+}
+function getTabularFields(value) {
+  if (value.length === 0 || !value.every(isObjectRecord)) {
+    return void 0;
+  }
+  const first = value[0];
+  const fields = Object.keys(first);
+  if (fields.length === 0) {
+    return void 0;
+  }
+  for (const row of value) {
+    const keys = Object.keys(row);
+    if (keys.length !== fields.length) {
+      return void 0;
+    }
+    for (const field of fields) {
+      if (!(field in row) || !isPrimitive(row[field])) {
+        return void 0;
+      }
+    }
+  }
+  return fields;
+}
+function encodePrimitive(value, delimiter) {
+  if (value === null) {
+    return "null";
+  }
+  if (typeof value === "number" || typeof value === "boolean") {
+    return String(value);
+  }
+  return encodeString(value, delimiter);
+}
+function encodeString(value, delimiter) {
+  if (isSafeUnquotedString(value, delimiter)) {
+    return value;
+  }
+  return JSON.stringify(value);
+}
+function encodeKey(key) {
+  if (/^[A-Z_][\w.]*$/i.test(key)) {
+    return key;
+  }
+  return JSON.stringify(key);
+}
+function isSafeUnquotedString(value, delimiter) {
+  if (value.length === 0 || value !== value.trim()) {
+    return false;
+  }
+  if (/^(?:true|false|null)$/i.test(value) || /^-?\d+(?:\.\d+)?(?:e[+-]?\d+)?$/i.test(value) || /^0\d+$/.test(value)) {
+    return false;
+  }
+  if (value.includes(":") || value.includes('"') || value.includes("\\") || value.includes(delimiter)) {
+    return false;
+  }
+  if (/[[\]{}]|[\u0000-\u001F]/.test(value) || value.startsWith("-")) {
+    return false;
+  }
+  return true;
+}
+function isPrimitive(value) {
+  return value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean";
+}
+function isObjectRecord(value) {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function isPlainObject(value) {
+  if (value === null || typeof value !== "object") {
+    return false;
+  }
+  const prototype = Object.getPrototypeOf(value);
+  return prototype === null || prototype === Object.prototype;
+}
+function line(depth, content, indent) {
+  return `${" ".repeat(depth * indent)}${content}`;
+}
+
+// src/token-compression.ts
+var TOKEN_REGEX = /[\p{L}\p{N}]+|[^\s]/gu;
+var TOKEN_NORMALIZE_REGEX = /[^\p{L}\p{N}]/gu;
+var DEDUPE_NORMALIZE_REGEX = /[^\p{L}\p{N}\s]/gu;
+var BOILERPLATE_REGEX = /copyright|all rights reserved|disclaimer|confidential/i;
+var MAX_SEGMENT_TOKENS = 120;
+var STOP_WORDS = /* @__PURE__ */ new Set([
+  "a",
+  "an",
+  "and",
+  "are",
+  "as",
+  "at",
+  "be",
+  "by",
+  "for",
+  "from",
+  "how",
+  "in",
+  "is",
+  "it",
+  "of",
+  "on",
+  "or",
+  "that",
+  "the",
+  "this",
+  "to",
+  "was",
+  "we",
+  "with",
+  "you",
+  "your",
+  "our"
+]);
+var DEFAULT_OPTIONS = {
+  min_tokens: 80,
+  max_sentences: 10,
+  target_reduction: 0.45
+};
+function tokenizeForHeuristicCompression(text) {
+  if (!text) {
+    return [];
+  }
+  return text.match(TOKEN_REGEX) ?? [];
+}
+function estimateHeuristicTokens(text) {
+  return tokenizeForHeuristicCompression(text).length;
+}
+function compressHeuristicText(input, options = {}) {
+  const inputText = input ?? "";
+  const inputTokens = estimateHeuristicTokens(inputText);
+  if (options.json_to_toon === true) {
+    const toonResult = tryJsonToToon(inputText);
+    if (toonResult) {
+      const toonOutputTokens = estimateHeuristicTokens(toonResult.output);
+      if (toonOutputTokens <= inputTokens) {
+        return {
+          ...toCompressionOutput(toonResult.output, inputTokens),
+          outputFormat: "toon"
+        };
+      }
+      return {
+        ...toCompressionOutput(inputText, inputTokens),
+        warnings: ["JSON-to-TOON skipped because TOON was not smaller than the original JSON."]
+      };
+    }
+    return {
+      ...toCompressionOutput(inputText, inputTokens),
+      warnings: ["JSON-to-TOON skipped because the input is not a complete valid JSON object or array."]
+    };
+  }
+  const analysis = preprocessContext(inputText);
+  if (inputTokens === 0) {
+    return toCompressionOutput(inputText, inputTokens);
+  }
+  const minTokens = options.min_tokens ?? DEFAULT_OPTIONS.min_tokens;
+  if (inputTokens <= minTokens) {
+    return toCompressionOutput(inputText, inputTokens);
+  }
+  const targetReduction = options.target_reduction ?? DEFAULT_OPTIONS.target_reduction;
+  const targetTokens = Math.max(1, Math.max(minTokens, Math.floor(inputTokens * (1 - targetReduction))));
+  const maxSentences = options.max_sentences ?? DEFAULT_OPTIONS.max_sentences;
+  const query = options.query ?? "";
+  const terms = queryTerms(query);
+  const scoredSentences = analysis.candidates.map((candidate) => ({
+    sentence: candidate.sentence,
+    index: candidate.index,
+    ...scoreSentenceCandidate(candidate, terms),
+    tokens: candidate.tokenCount
+  }));
+  const selected = [];
+  const selectedIndices = /* @__PURE__ */ new Set();
+  const coveredTerms = /* @__PURE__ */ new Set();
+  let selectedTokens = 0;
+  while (selected.length < maxSentences) {
+    let bestCandidate;
+    let bestAdjustedScore = Number.NEGATIVE_INFINITY;
+    for (const candidate of scoredSentences) {
+      if (selectedIndices.has(candidate.index)) {
+        continue;
+      }
+      const underBudget = selectedTokens + candidate.tokens <= targetTokens;
+      const forceTopCandidate = selected.length === 0;
+      const forceRelevantSecond = selected.length === 1 && candidate.overlapCount > 0;
+      if (!underBudget && !forceTopCandidate && !forceRelevantSecond) {
+        continue;
+      }
+      let uncoveredOverlapCount = 0;
+      for (const term of candidate.overlapTerms) {
+        if (!coveredTerms.has(term)) {
+          uncoveredOverlapCount += 1;
+        }
+      }
+      const adjustedScore = candidate.score + uncoveredOverlapCount * 1.2;
+      if (adjustedScore > bestAdjustedScore || adjustedScore === bestAdjustedScore && bestCandidate && isBetterTieBreak(candidate, bestCandidate)) {
+        bestAdjustedScore = adjustedScore;
+        bestCandidate = candidate;
+      }
+    }
+    if (!bestCandidate) {
+      break;
+    }
+    selected.push(bestCandidate);
+    selectedIndices.add(bestCandidate.index);
+    selectedTokens += bestCandidate.tokens;
+    for (const term of bestCandidate.overlapTerms) {
+      coveredTerms.add(term);
+    }
+  }
+  selected.sort((left, right) => left.index - right.index);
+  let output = selected.map((item) => item.sentence).join(" ");
+  if (!output) {
+    output = tokenizeForHeuristicCompression(inputText).slice(0, targetTokens).join(" ");
+  } else {
+    const outputTokens = tokenizeForHeuristicCompression(output);
+    if (outputTokens.length > targetTokens) {
+      output = outputTokens.slice(0, targetTokens).join(" ");
+    }
+  }
+  return toCompressionOutput(output, inputTokens);
+}
+function toCompressionOutput(output, inputTokens) {
+  const outputTokens = estimateHeuristicTokens(output);
+  const tokensSaved = Math.max(0, inputTokens - outputTokens);
+  return {
+    output,
+    inputTokens,
+    outputTokens,
+    tokensSaved,
+    compressionRatio: outputTokens === 0 ? 0 : inputTokens / outputTokens
+  };
+}
+function isBetterTieBreak(candidate, bestCandidate) {
+  if (candidate.overlapCount !== bestCandidate.overlapCount) {
+    return candidate.overlapCount > bestCandidate.overlapCount;
+  }
+  if (candidate.tokens !== bestCandidate.tokens) {
+    return candidate.tokens < bestCandidate.tokens;
+  }
+  return candidate.index < bestCandidate.index;
+}
+function normalizeForDeduping(sentence) {
+  return sentence.toLowerCase().replace(DEDUPE_NORMALIZE_REGEX, " ").replace(/\s+/g, " ").trim();
+}
+function splitIntoSentences(text) {
+  return text.split(/(?<=[.!?])\s+|\n+/g).map((sentence) => sentence.trim()).filter(Boolean);
+}
+function chunkTokens(tokens, chunkSize) {
+  const chunks = [];
+  for (let index = 0; index < tokens.length; index += chunkSize) {
+    chunks.push(tokens.slice(index, index + chunkSize));
+  }
+  return chunks;
+}
+function splitOversizedSentence(sentence, maxTokens = MAX_SEGMENT_TOKENS) {
+  const rawTokens = tokenizeForHeuristicCompression(sentence);
+  if (rawTokens.length <= maxTokens) {
+    return [sentence];
+  }
+  const commaSplit = sentence.split(/(?<=[,;:])\s+|\s+-\s+/g).map((segment) => segment.trim()).filter(Boolean);
+  if (commaSplit.length > 1) {
+    return commaSplit.flatMap((segment) => {
+      if (estimateHeuristicTokens(segment) <= maxTokens) {
+        return [segment];
+      }
+      return chunkTokens(tokenizeForHeuristicCompression(segment), maxTokens).map((chunk) => chunk.join(" "));
+    });
+  }
+  return chunkTokens(rawTokens, maxTokens).map((chunk) => chunk.join(" "));
+}
+function normalizeToken(token) {
+  return token.replace(TOKEN_NORMALIZE_REGEX, "").trim();
+}
+function tokenizeNormalized(text) {
+  return tokenizeForHeuristicCompression(text.toLowerCase()).map(normalizeToken).filter(Boolean);
+}
+function queryTerms(query) {
+  return new Set(tokenizeNormalized(query).filter((term) => term.length >= 3 && !STOP_WORDS.has(term)));
+}
+function preprocessContext(context) {
+  const originalTokens = estimateHeuristicTokens(context);
+  const dedupeSet = /* @__PURE__ */ new Set();
+  const candidates = [];
+  for (const sentence of splitIntoSentences(context)) {
+    for (const segment of splitOversizedSentence(sentence)) {
+      const normalized = normalizeForDeduping(segment);
+      if (!normalized || dedupeSet.has(normalized)) {
+        continue;
+      }
+      dedupeSet.add(normalized);
+      const rawTokens = tokenizeForHeuristicCompression(segment);
+      const normalizedTerms = rawTokens.map((token) => normalizeToken(token.toLowerCase())).filter(Boolean);
+      const lowered = segment.toLowerCase();
+      candidates.push({
+        sentence: segment,
+        index: candidates.length,
+        tokenCount: rawTokens.length,
+        uniqueTerms: new Set(normalizedTerms),
+        isBoilerplate: BOILERPLATE_REGEX.test(lowered)
+      });
+    }
+  }
+  return { originalTokens, candidates };
+}
+function scoreSentenceCandidate(candidate, terms) {
+  if (candidate.tokenCount === 0) {
+    return {
+      score: 0,
+      overlapCount: 0,
+      overlapTerms: []
+    };
+  }
+  const overlapTerms = [];
+  let overlap = 0;
+  for (const term of terms) {
+    if (candidate.uniqueTerms.has(term)) {
+      overlap += 1;
+      overlapTerms.push(term);
+    }
+  }
+  const overlapScore = overlap * 2;
+  const densityScore = terms.size > 0 ? overlap / terms.size : 0;
+  const lengthScore = Math.min(candidate.tokenCount, 40) / 40;
+  const boilerplatePenalty = candidate.isBoilerplate ? 0.5 : 0;
+  return {
+    score: overlapScore + densityScore + lengthScore - boilerplatePenalty,
+    overlapCount: overlap,
+    overlapTerms
+  };
+}
+
+// src/code-compaction.ts
+function compactCode(input, options = {}) {
+  const inputText = input ?? "";
+  const removeComments = options.remove_comments ?? true;
+  const trimIndentation = options.trim_indentation ?? true;
+  const collapseBlankLines = options.collapse_blank_lines ?? true;
+  let output = inputText.replace(/\r\n?/g, "\n");
+  if (removeComments) {
+    output = stripCodeComments(output);
+  }
+  output = output.split("\n").map((line2) => line2.replace(/[ \t]+$/g, "")).join("\n");
+  if (trimIndentation) {
+    output = stripCommonIndent(output);
+  }
+  if (collapseBlankLines) {
+    output = output.split("\n").filter((line2) => line2.trim().length > 0).join("\n");
+  }
+  output = output.trim();
+  const inputTokens = estimateHeuristicTokens(inputText);
+  const outputTokens = estimateHeuristicTokens(output);
+  return {
+    output,
+    inputTokens,
+    outputTokens,
+    tokensSaved: Math.max(0, inputTokens - outputTokens),
+    compressionRatio: outputTokens === 0 ? 0 : inputTokens / outputTokens
+  };
+}
+function stripCommonIndent(input) {
+  const lines = input.split("\n");
+  const nonEmptyLines = lines.filter((line2) => line2.trim().length > 0);
+  if (nonEmptyLines.length === 0) {
+    return "";
+  }
+  const commonIndent = Math.min(
+    ...nonEmptyLines.map((line2) => {
+      const match = line2.match(/^[ \t]*/);
+      return match?.[0].length ?? 0;
+    })
+  );
+  if (commonIndent === 0) {
+    return input;
+  }
+  return lines.map((line2) => line2.slice(commonIndent)).join("\n");
+}
+function stripCodeComments(input) {
+  let output = "";
+  let index = 0;
+  let quote;
+  let escaped = false;
+  let atLineStart = true;
+  while (index < input.length) {
+    const char = input[index];
+    const next = input[index + 1];
+    if (quote) {
+      output += char;
+      if (escaped) {
+        escaped = false;
+      } else if (char === "\\") {
+        escaped = true;
+      } else if (char === quote) {
+        quote = void 0;
+      }
+      atLineStart = char === "\n";
+      index += 1;
+      continue;
+    }
+    if (char === '"' || char === "'" || char === "`") {
+      quote = char;
+      output += char;
+      atLineStart = false;
+      index += 1;
+      continue;
+    }
+    if (atLineStart && char === "#" && next === "!") {
+      while (index < input.length && input[index] !== "\n") {
+        output += input[index];
+        index += 1;
+      }
+      continue;
+    }
+    if (char === "/" && next === "/") {
+      index += 2;
+      while (index < input.length && input[index] !== "\n") {
+        index += 1;
+      }
+      continue;
+    }
+    if (char === "/" && next === "*") {
+      index += 2;
+      while (index < input.length && !(input[index] === "*" && input[index + 1] === "/")) {
+        if (input[index] === "\n") {
+          output += "\n";
+        }
+        index += 1;
+      }
+      index += input[index] === "*" ? 2 : 0;
+      continue;
+    }
+    if (char === "#" && (next === " " || next === "	")) {
+      index += 1;
+      while (index < input.length && input[index] !== "\n") {
+        index += 1;
+      }
+      continue;
+    }
+    output += char;
+    atLineStart = char === "\n" || atLineStart && (char === " " || char === "	");
+    index += 1;
+  }
+  return output;
+}
+
 // src/renderer/renderer.ts
 function renderSections(asset, options = {}) {
   const { variables = {}, strict = false } = options;
-  const optionalVariables = getContextInputs(asset).filter((input) => input.optional === true).map((input) => input.name);
+  const contextInputs = getContextInputs(asset);
+  const optionalVariables = contextInputs.filter((input) => input.optional === true).map((input) => input.name);
+  const contextInputsByName = new Map(contextInputs.map((input) => [input.name, input]));
+  const compressionCache = /* @__PURE__ */ new Map();
+  const createTransformVariable = (template) => {
+    return ({ name, value, modifier, match }) => {
+      const input = contextInputsByName.get(name);
+      const configuredHeuristic = input?.compression?.heuristic;
+      const configuredCode = input?.compression?.code;
+      const shouldCompactCode = modifier === "compact" || modifier === "code" || configuredCode?.enabled === true;
+      const shouldCompress = modifier === "compress" || configuredHeuristic?.enabled === true;
+      if (modifier === "toon") {
+        const result3 = toToonPlaceholder(value);
+        if (!result3) {
+          options.compression?.onCompressionWarning?.(
+            `POK031: JSON-to-TOON skipped for placeholder "${name}" because the value is not a complete valid JSON object or array.`
+          );
+          return value;
+        }
+        if (result3.tokensSaved > 0 || result3.output !== value) {
+          options.compression?.onHeuristicCompression?.({
+            scope: "placeholder",
+            variable: name,
+            ...result3
+          });
+        }
+        return result3.output;
+      }
+      if (shouldCompactCode) {
+        const result3 = compactCode(value, toCodeCompactionOptions(configuredCode));
+        if (result3.tokensSaved > 0 || result3.output !== value) {
+          options.compression?.onCodeCompaction?.({
+            scope: "placeholder",
+            variable: name,
+            ...result3
+          });
+        }
+        return result3.output;
+      }
+      if (!shouldCompress) {
+        return value;
+      }
+      const query = resolveHeuristicQuery(configuredHeuristic, variables, template, match);
+      const compressionOptions = {
+        min_tokens: configuredHeuristic?.min_tokens,
+        max_sentences: configuredHeuristic?.max_sentences,
+        target_reduction: configuredHeuristic?.target_reduction,
+        query,
+        json_to_toon: configuredHeuristic?.json_to_toon
+      };
+      const cacheKey = JSON.stringify([name, value, compressionOptions]);
+      const cached = compressionCache.get(cacheKey);
+      if (cached !== void 0) {
+        return cached;
+      }
+      const result2 = compressHeuristicText(value, compressionOptions);
+      compressionCache.set(cacheKey, result2.output);
+      reportHeuristicCompressionWarnings(options, result2, `placeholder "${name}"`);
+      if (result2.tokensSaved > 0 || result2.output !== value) {
+        options.compression?.onHeuristicCompression?.({
+          scope: "placeholder",
+          variable: name,
+          ...result2
+        });
+      }
+      return result2.output;
+    };
+  };
   const result = {};
   if (asset.sections.system_instructions) {
     result.system_instructions = interpolate(
       asset.sections.system_instructions,
       variables,
-      { strict, optionalVariables }
+      {
+        strict,
+        optionalVariables,
+        transformVariable: createTransformVariable(asset.sections.system_instructions)
+      }
     );
   }
   if (asset.sections.prompt_template) {
     result.prompt_template = interpolate(
       asset.sections.prompt_template,
       variables,
-      { strict, optionalVariables }
+      {
+        strict,
+        optionalVariables,
+        transformVariable: createTransformVariable(asset.sections.prompt_template)
+      }
     );
   }
   return result;
 }
+function resolveHeuristicQuery(options, variables, template, match) {
+  if (options?.query !== void 0) {
+    return options.query;
+  }
+  if (options?.query_variable && variables[options.query_variable] !== void 0) {
+    return variables[options.query_variable];
+  }
+  return template.replace(match, " ");
+}
+function toToonPlaceholder(value) {
+  const toon = tryJsonToToon(value);
+  if (!toon) {
+    return void 0;
+  }
+  const inputTokens = estimateHeuristicTokens(value);
+  const outputTokens = estimateHeuristicTokens(toon.output);
+  return {
+    output: toon.output,
+    inputTokens,
+    outputTokens,
+    tokensSaved: Math.max(0, inputTokens - outputTokens),
+    compressionRatio: outputTokens === 0 ? 0 : inputTokens / outputTokens,
+    outputFormat: "toon"
+  };
+}
+function toCodeCompactionOptions(options) {
+  return {
+    remove_comments: options?.remove_comments,
+    trim_indentation: options?.trim_indentation,
+    collapse_blank_lines: options?.collapse_blank_lines
+  };
+}
+function reportHeuristicCompressionWarnings(options, result, scope) {
+  for (const warning of result.warnings ?? []) {
+    options.compression?.onCompressionWarning?.(`POK031: ${warning} Scope: ${scope}.`);
+  }
+}
 
 // src/overrides/apply-overrides.ts
 function applyOverrides(asset, options = {}) {
@@ -388,6 +1109,14 @@ function mergeOverride(base, override) {
       thetokencompany: mergeRecordBlock(
         result.compression?.thetokencompany,
         override.compression.thetokencompany
+      ),
+      heuristic: mergeRecordBlock(
+        result.compression?.heuristic,
+        override.compression.heuristic
+      ),
+      code: mergeRecordBlock(
+        result.compression?.code,
+        override.compression.code
       )
     };
   }
@@ -621,8 +1350,25 @@ var TheTokenCompanyCompressionSchema = import_zod.z.object({
   model: import_zod.z.string().min(1).optional(),
   aggressiveness: import_zod.z.number().min(0).max(1).optional()
 });
+var HeuristicCompressionSchema = import_zod.z.object({
+  enabled: import_zod.z.boolean().optional(),
+  min_tokens: import_zod.z.number().int().positive().optional(),
+  max_sentences: import_zod.z.number().int().positive().optional(),
+  target_reduction: import_zod.z.number().min(0).max(1).optional(),
+  query: import_zod.z.string().optional(),
+  query_variable: import_zod.z.string().min(1).optional(),
+  json_to_toon: import_zod.z.boolean().optional()
+});
+var CodeCompactionSchema = import_zod.z.object({
+  enabled: import_zod.z.boolean().optional(),
+  remove_comments: import_zod.z.boolean().optional(),
+  trim_indentation: import_zod.z.boolean().optional(),
+  collapse_blank_lines: import_zod.z.boolean().optional()
+});
 var CompressionSchema = import_zod.z.object({
-  thetokencompany: TheTokenCompanyCompressionSchema.optional()
+  thetokencompany: TheTokenCompanyCompressionSchema.optional(),
+  heuristic: HeuristicCompressionSchema.optional(),
+  code: CodeCompactionSchema.optional()
 });
 var OpenAICacheSchema = import_zod.z.object({
   prompt_cache_key: import_zod.z.string().min(1).optional(),
@@ -662,12 +1408,21 @@ var ContextBuiltInValidatorSchema = import_zod.z.union([
     return_message: import_zod.z.string().optional()
   })
 ]);
+var ContextInputCompressionSchema = import_zod.z.union([
+  import_zod.z.literal("heuristic"),
+  import_zod.z.literal("code"),
+  import_zod.z.object({
+    heuristic: HeuristicCompressionSchema.optional(),
+    code: CodeCompactionSchema.optional()
+  })
+]);
 var ContextInputDefinitionObjectSchema = import_zod.z.object({
   name: import_zod.z.string(),
   optional: import_zod.z.boolean().optional(),
   warnings: import_zod.z.boolean().optional(),
   max_size: import_zod.z.number().int().positive().optional(),
   trim: import_zod.z.union([import_zod.z.boolean(), import_zod.z.enum(["start", "end", "both"])]).optional(),
+  compression: ContextInputCompressionSchema.optional(),
   allow_regex: ContextRegexSchema.optional(),
   deny_regex: ContextRegexSchema.optional(),
   non_empty: ContextBuiltInValidatorSchema.optional(),
@@ -770,9 +1525,9 @@ function assertRegexFrontMatterQuoting(content, filePath) {
     return;
   }
   let pendingRegex;
-  for (const [index, line] of frontMatter.lines.entries()) {
+  for (const [index, line2] of frontMatter.lines.entries()) {
     const lineNumber = frontMatter.startLine + index;
-    const regexMatch = line.match(REGEX_FIELD);
+    const regexMatch = line2.match(REGEX_FIELD);
     if (regexMatch?.groups) {
       const field = regexMatch.groups.field;
       const indent2 = regexMatch.groups.indent.length;
@@ -785,15 +1540,15 @@ function assertRegexFrontMatterQuoting(content, filePath) {
     if (!pendingRegex) {
       continue;
     }
-    if (line.trim() === "" || line.trimStart().startsWith("#")) {
+    if (line2.trim() === "" || line2.trimStart().startsWith("#")) {
       continue;
     }
-    const indent = line.match(/^\s*/)?.[0].length ?? 0;
+    const indent = line2.match(/^\s*/)?.[0].length ?? 0;
     if (indent <= pendingRegex.indent) {
       pendingRegex = void 0;
       continue;
     }
-    const patternMatch = line.match(PATTERN_FIELD);
+    const patternMatch = line2.match(PATTERN_FIELD);
     if (!patternMatch?.groups) {
       continue;
     }
@@ -888,8 +1643,8 @@ function extractSections(body) {
   let currentKey = null;
   let currentLines = [];
   let foundAnyH1 = false;
-  for (const line of lines) {
-    const h1Match = line.match(/^#\s+(.+)$/);
+  for (const line2 of lines) {
+    const h1Match = line2.match(/^#\s+(.+)$/);
     if (h1Match) {
       if (currentKey) {
         sections[currentKey] = currentLines.join("\n").trim();
@@ -899,7 +1654,7 @@ function extractSections(body) {
       currentKey = SECTION_MAP[heading] ?? null;
       currentLines = [];
     } else {
-      currentLines.push(line);
+      currentLines.push(line2);
     }
   }
   if (currentKey) {
@@ -1232,7 +1987,9 @@ function mergeCompression(base, local) {
   const merged = {
     ...base ?? {},
     ...local ?? {},
-    thetokencompany: mergeRecordBlock2(base?.thetokencompany, local?.thetokencompany)
+    thetokencompany: mergeRecordBlock2(base?.thetokencompany, local?.thetokencompany),
+    heuristic: mergeRecordBlock2(base?.heuristic, local?.heuristic),
+    code: mergeRecordBlock2(base?.code, local?.code)
   };
   removeEmptyProviderBlocks(merged);
   return Object.keys(merged).length > 0 ? merged : void 0;
@@ -1486,33 +2243,152 @@ function compactHistoryForPrompt(asset, runtime) {
 // src/compression.ts
 var THETOKENCOMPANY_DEFAULT_MODEL = "bear-2";
 var THETOKENCOMPANY_DEFAULT_BASE_URL = "https://api.thetokencompany.com";
+var HEURISTIC_COMPRESSION_MODEL = "local-heuristic-v1";
+var CODE_COMPACTION_MODEL = "local-code-compactor-v1";
+function summarizePromptCompression(compression = []) {
+  const totals = compression.reduce(
+    (accumulator, result) => {
+      accumulator.inputTokens += result.inputTokens;
+      accumulator.outputTokens += result.outputTokens;
+      accumulator.tokensSaved += result.tokensSaved;
+      return accumulator;
+    },
+    {
+      inputTokens: 0,
+      outputTokens: 0,
+      tokensSaved: 0
+    }
+  );
+  return {
+    steps: compression.length,
+    inputTokens: totals.inputTokens,
+    outputTokens: totals.outputTokens,
+    tokensSaved: totals.tokensSaved,
+    reductionRatio: totals.inputTokens === 0 ? 0 : totals.tokensSaved / totals.inputTokens
+  };
+}
 async function applyPromptCompressionForRender(asset, runtime) {
-  const config = asset.compression?.thetokencompany;
-  if (config?.enabled !== true || !asset.sections.prompt_template) {
-    return { asset, runtime, compression: [] };
-  }
+  const theTokenCompanyConfig = asset.compression?.thetokencompany;
+  const heuristicConfig = asset.compression?.heuristic;
+  const codeConfig = asset.compression?.code;
+  const hasTheTokenCompanyCompression = theTokenCompanyConfig?.enabled === true && Boolean(asset.sections.prompt_template);
+  const hasHeuristicPromptCompression = heuristicConfig?.enabled === true && Boolean(asset.sections.prompt_template);
+  const hasCodePromptCompaction = codeConfig?.enabled === true && Boolean(asset.sections.prompt_template);
+  const hasPlaceholderCompression = usesPlaceholderCompression(asset);
+  if (!hasTheTokenCompanyCompression && !hasHeuristicPromptCompression && !hasCodePromptCompaction && !hasPlaceholderCompression) {
+    return { asset, runtime, compression: [], warnings: [] };
+  }
+  const compression = [];
+  const warnings = [];
   const sections = renderSections(asset, {
     variables: runtime.variables,
-    strict: runtime.strict
-  });
-  if (!sections.prompt_template) {
-    return { asset, runtime, compression: [] };
-  }
-  const model = config.model ?? THETOKENCOMPANY_DEFAULT_MODEL;
-  const result = await compressWithTheTokenCompany(sections.prompt_template, {
-    apiKey: runtime.theTokenCompany?.apiKey,
-    baseURL: runtime.theTokenCompany?.baseURL,
-    fetch: runtime.theTokenCompany?.fetch,
-    model,
-    aggressiveness: config.aggressiveness
+    strict: runtime.strict,
+    compression: {
+      onHeuristicCompression: (event) => {
+        compression.push({
+          provider: "heuristic",
+          model: HEURISTIC_COMPRESSION_MODEL,
+          inputTokens: event.inputTokens,
+          outputTokens: event.outputTokens,
+          tokensSaved: event.tokensSaved,
+          compressionRatio: event.compressionRatio,
+          scope: event.scope,
+          variable: event.variable,
+          outputFormat: event.outputFormat
+        });
+      },
+      onCodeCompaction: (event) => {
+        compression.push({
+          provider: "code",
+          model: CODE_COMPACTION_MODEL,
+          inputTokens: event.inputTokens,
+          outputTokens: event.outputTokens,
+          tokensSaved: event.tokensSaved,
+          compressionRatio: event.compressionRatio,
+          scope: event.scope,
+          variable: event.variable,
+          outputFormat: "code"
+        });
+      },
+      onCompressionWarning: (warning) => {
+        warnings.push(warning);
+      }
+    }
   });
+  let promptTemplate = sections.prompt_template;
+  if (hasCodePromptCompaction && promptTemplate) {
+    if (hasHeuristicPromptCompression) {
+      warnings.push(
+        "POK032: Local heuristic prompt compression skipped because compression.code is enabled for the prompt template."
+      );
+    }
+    const result = compactCode(promptTemplate, toCodeCompactionOptions2(codeConfig));
+    promptTemplate = result.output;
+    if (result.tokensSaved > 0 || result.output !== sections.prompt_template) {
+      compression.push({
+        provider: "code",
+        model: CODE_COMPACTION_MODEL,
+        inputTokens: result.inputTokens,
+        outputTokens: result.outputTokens,
+        tokensSaved: result.tokensSaved,
+        compressionRatio: result.compressionRatio,
+        scope: "prompt_template",
+        outputFormat: "code"
+      });
+    }
+  } else if (hasHeuristicPromptCompression && promptTemplate) {
+    const result = compressHeuristicText(promptTemplate, {
+      min_tokens: heuristicConfig?.min_tokens,
+      max_sentences: heuristicConfig?.max_sentences,
+      target_reduction: heuristicConfig?.target_reduction,
+      query: resolveHeuristicPromptQuery(heuristicConfig, runtime.variables, sections.system_instructions),
+      json_to_toon: heuristicConfig?.json_to_toon
+    });
+    promptTemplate = result.output;
+    reportHeuristicCompressionWarnings2(warnings, result.warnings, "prompt template");
+    if (result.tokensSaved > 0 || result.output !== sections.prompt_template) {
+      compression.push({
+        provider: "heuristic",
+        model: HEURISTIC_COMPRESSION_MODEL,
+        inputTokens: result.inputTokens,
+        outputTokens: result.outputTokens,
+        tokensSaved: result.tokensSaved,
+        compressionRatio: result.compressionRatio,
+        scope: "prompt_template",
+        outputFormat: result.outputFormat
+      });
+    }
+  }
+  if (hasTheTokenCompanyCompression && hasCodePromptCompaction && promptTemplate) {
+    warnings.push(
+      "POK033: TheTokenCompany compression skipped because compression.code is enabled; code is compacted locally and not text-compressed."
+    );
+  } else if (hasTheTokenCompanyCompression && promptTemplate) {
+    const model = theTokenCompanyConfig?.model ?? THETOKENCOMPANY_DEFAULT_MODEL;
+    const result = await compressWithTheTokenCompany(promptTemplate, {
+      apiKey: runtime.theTokenCompany?.apiKey,
+      baseURL: runtime.theTokenCompany?.baseURL,
+      fetch: runtime.theTokenCompany?.fetch,
+      model,
+      aggressiveness: theTokenCompanyConfig?.aggressiveness
+    });
+    promptTemplate = result.output;
+    compression.push({
+      provider: "thetokencompany",
+      model,
+      inputTokens: result.input_tokens,
+      outputTokens: result.output_tokens,
+      tokensSaved: result.tokens_saved,
+      compressionRatio: result.compression_ratio
+    });
+  }
   return {
     asset: {
       ...asset,
       sections: {
         ...asset.sections,
         ...sections,
-        prompt_template: result.output
+        ...promptTemplate !== void 0 ? { prompt_template: promptTemplate } : {}
       }
     },
     runtime: {
@@ -1520,16 +2396,45 @@ async function applyPromptCompressionForRender(asset, runtime) {
       variables: {},
       strict: false
     },
-    compression: [{
-      provider: "thetokencompany",
-      model,
-      inputTokens: result.input_tokens,
-      outputTokens: result.output_tokens,
-      tokensSaved: result.tokens_saved,
-      compressionRatio: result.compression_ratio
-    }]
+    compression,
+    warnings
+  };
+}
+function usesPlaceholderCompression(asset) {
+  const hasContextInputCompression = getContextInputs(asset).some(
+    (input) => input.compression?.heuristic?.enabled === true || input.compression?.code?.enabled === true
+  );
+  if (hasContextInputCompression) {
+    return true;
+  }
+  return Boolean(
+    asset.sections.system_instructions && hasCompressionModifier(asset.sections.system_instructions) || asset.sections.prompt_template && hasCompressionModifier(asset.sections.prompt_template)
+  );
+}
+function hasCompressionModifier(template) {
+  return /\{\{\s*[a-zA-Z_][a-zA-Z0-9_]*\s*\|\s*(?:compress|toon|compact|code)\s*\}\}/.test(template);
+}
+function resolveHeuristicPromptQuery(options, variables, systemInstructions) {
+  if (options?.query !== void 0) {
+    return options.query;
+  }
+  if (options?.query_variable && variables?.[options.query_variable] !== void 0) {
+    return variables[options.query_variable];
+  }
+  return systemInstructions;
+}
+function toCodeCompactionOptions2(options) {
+  return {
+    remove_comments: options?.remove_comments,
+    trim_indentation: options?.trim_indentation,
+    collapse_blank_lines: options?.collapse_blank_lines
   };
 }
+function reportHeuristicCompressionWarnings2(warnings, compressionWarnings, scope) {
+  for (const warning of compressionWarnings ?? []) {
+    warnings.push(`POK031: ${warning} Scope: ${scope}.`);
+  }
+}
 async function compressWithTheTokenCompany(input, options) {
   const apiKey = options.apiKey ?? getEnv("THETOKENCOMPANY_API_KEY") ?? getEnv("TTC_API_KEY");
   if (!apiKey) {
@@ -1617,7 +2522,14 @@ function withPromptInputSupport(adapter) {
       history: compactHistoryForPrompt(resolvedForRender, runtime)
     });
     const request = adapter.render(prepared.asset, prepared.runtime);
-    return prepared.compression.length > 0 ? { ...request, compression: prepared.compression } : request;
+    return prepared.compression.length > 0 || prepared.warnings.length > 0 ? {
+      ...request,
+      ...prepared.compression.length > 0 ? {
+        compression: prepared.compression,
+        compressionSummary: summarizePromptCompression(prepared.compression)
+      } : {},
+      ...prepared.warnings.length > 0 ? { warnings: prepared.warnings } : {}
+    } : request;
   };
   return {
     ...adapter,