promptopskit 0.8.2 → 0.9.0

package/README.md

@@ -300,6 +300,7 @@ For no-backend compression, use the local heuristic compressor:
 compression:
   heuristic:
     enabled: true
+    mode: conservative
     query_variable: user_question
     json_to_toon: true
 ```
@@ -320,7 +321,7 @@ Payload: {{ json_payload | toon }}
 Source: {{ source_code | compact }}
 ```
 
-If `json_to_toon: true` or `{{ value | toon }}` cannot parse a complete JSON object or array, PromptOpsKit preserves the original value and returns a `POK031` warning. When `compression.code.enabled: true`, PromptOpsKit skips TheTokenCompany prompt-template compression and returns `POK033` so code is not text-compressed by a backend.
+The local heuristic compressor is extractive. In its default conservative mode it preserves whole source sentences, skips low-confidence matches, includes nearby context when configured capacity allows, and leaves structured blocks unchanged. If `json_to_toon: true` or `{{ value | toon }}` cannot parse a complete JSON object or array, PromptOpsKit preserves the original value and returns a `POK031` warning. When `compression.code.enabled: true`, PromptOpsKit skips TheTokenCompany prompt-template compression and returns `POK033` so code is not text-compressed by a backend.
 
 Credit: the local heuristic approach is based on Jason Kneen's [open-thetokenco](https://github.com/jasonkneen/open-thetokenco/tree/main).
 TOON preprocessing uses a local encode-only implementation inspired by the MIT-licensed [TOON project](https://github.com/toon-format/toon) by Johann Schopplich, without adding `@toon-format/toon` as a runtime dependency.

package/SKILL.md

@@ -485,15 +485,19 @@ language context and no backend call should be made:
 compression:
   heuristic:
     enabled: true
+    mode: conservative
     min_tokens: 120
     max_sentences: 8
     target_reduction: 0.45
     query_variable: user_question
 ```
 
-The heuristic compressor deduplicates text, ranks sentences against `query`,
-`query_variable`, or system instructions, and keeps the highest-scoring
-sentences inside the token budget. It is heuristic, so use it for context where
+The heuristic compressor deduplicates adjacent repeated sentences, ranks exact
+source sentences against `query`, `query_variable`, or system instructions, and
+keeps the highest-scoring sentences inside the token budget. The default
+`mode: conservative` skips low-confidence compression, preserves adjacent
+context when `max_sentences` allows, avoids token-level truncation, and leaves
+structured blocks unchanged. It is heuristic, so use it for context where
 extractive reduction is acceptable, not for code or strict structured payloads.
 
 Use TOON preprocessing for complete JSON object/array inputs:
@@ -550,6 +554,7 @@ context:
       compression:
         heuristic:
           enabled: true
+          mode: conservative
           query_variable: user_question
     - name: payload
       compression:
@@ -570,7 +575,8 @@ Source: {{ source_code | compact }}
 
 Placeholder modifiers are single-token shortcuts. Use
 `context.inputs[].compression` when a placeholder needs options such as
-`query_variable`, `json_to_toon`, or `remove_comments: false`.
+`query_variable`, `mode`, `preserve_neighbors`, `fail_on_low_confidence`,
+`json_to_toon`, or `remove_comments: false`.
 
 Render results may include:
 

package/dist/{chunk-ZVCKP4EM.js → chunk-AAQNCAVA.js}

@@ -1,11 +1,11 @@
 import {
   openaiAdapter
-} from "./chunk-PDVUVZAU.js";
+} from "./chunk-SVNQNMMV.js";
 import {
   applyRawProviderBody,
   resolveAssetForProvider,
   withPromptInputSupport
-} from "./chunk-RVAFVWR2.js";
+} from "./chunk-NS6OTKY2.js";
 
 // src/providers/openrouter.ts
 var openrouterAdapter = withPromptInputSupport({
@@ -58,4 +58,4 @@ var openrouterAdapter = withPromptInputSupport({
 export {
   openrouterAdapter
 };
-//# sourceMappingURL=chunk-ZVCKP4EM.js.map
+//# sourceMappingURL=chunk-AAQNCAVA.js.map

package/dist/{chunk-5XH2GROP.js → chunk-E2AIIPQC.js}

@@ -4,7 +4,7 @@ import {
   renderSections,
   resolveAssetForProvider,
   withPromptInputSupport
-} from "./chunk-RVAFVWR2.js";
+} from "./chunk-NS6OTKY2.js";
 
 // src/providers/openai-responses.ts
 var openaiResponsesAdapter = withPromptInputSupport({
@@ -112,4 +112,4 @@ var openaiResponsesAdapter = withPromptInputSupport({
 export {
   openaiResponsesAdapter
 };
-//# sourceMappingURL=chunk-5XH2GROP.js.map
+//# sourceMappingURL=chunk-E2AIIPQC.js.map

package/dist/{chunk-UQLAWU67.js → chunk-FJ3D76IV.js}

@@ -4,7 +4,7 @@ import {
   renderSections,
   resolveAssetForProvider,
   withPromptInputSupport
-} from "./chunk-RVAFVWR2.js";
+} from "./chunk-NS6OTKY2.js";
 
 // src/providers/gemini.ts
 var geminiAdapter = withPromptInputSupport({
@@ -120,4 +120,4 @@ var geminiAdapter = withPromptInputSupport({
 export {
   geminiAdapter
 };
-//# sourceMappingURL=chunk-UQLAWU67.js.map
+//# sourceMappingURL=chunk-FJ3D76IV.js.map

package/dist/{chunk-RVAFVWR2.js → chunk-NS6OTKY2.js}

@@ -1,7 +1,7 @@
 import {
   loadPromptFile,
   parsePrompt
-} from "./chunk-E4GEKWVP.js";
+} from "./chunk-VOXMOGU5.js";
 
 // src/renderer/interpolate.ts
 var VARIABLE_RE = /\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)(?:\s*\|\s*(compress|toon|compact|code))?\s*\}\}/g;
@@ -79,7 +79,10 @@ function normalizeContextInputCompression(value) {
       target_reduction: value.heuristic.target_reduction,
       query: value.heuristic.query,
       query_variable: value.heuristic.query_variable,
-      json_to_toon: value.heuristic.json_to_toon
+      json_to_toon: value.heuristic.json_to_toon,
+      mode: value.heuristic.mode,
+      preserve_neighbors: value.heuristic.preserve_neighbors,
+      fail_on_low_confidence: value.heuristic.fail_on_low_confidence
     }
   };
   const code = normalizeCodeCompression(value.code);
@@ -584,7 +587,10 @@ function line(depth, content, indent) {
 var TOKEN_REGEX = /[\p{L}\p{N}]+|[^\s]/gu;
 var TOKEN_NORMALIZE_REGEX = /[^\p{L}\p{N}]/gu;
 var DEDUPE_NORMALIZE_REGEX = /[^\p{L}\p{N}\s]/gu;
-var BOILERPLATE_REGEX = /copyright|all rights reserved|disclaimer|confidential/i;
+var BOILERPLATE_REGEX = /copyright|all rights reserved|disclaimer/i;
+var STRUCTURED_BLOCK_REGEX = /(^|\n)\s*```|(^|\n)\s*\|.+\|\s*(\n|$)/;
+var PROTECTED_SIGNAL_REGEX = /\b(?:must|shall|should|required|requires|requirement|never|do not|don't|only|except|unless|however|but|constraint|constraints|safety|security|deadline|sla)\b|output\s+format/i;
+var EVIDENCE_SIGNAL_REGEX = /https?:\/\/|\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b|[$€£]\s?\d|\b\d+(?:[.,:/-]\d+)*%?\b|\b[A-Z]{2,}[-_A-Z0-9]*\b|\b[A-Z]\d\b/i;
 var MAX_SEGMENT_TOKENS = 120;
 var STOP_WORDS = /* @__PURE__ */ new Set([
   "a",
@@ -618,7 +624,8 @@ var STOP_WORDS = /* @__PURE__ */ new Set([
 var DEFAULT_OPTIONS = {
   min_tokens: 80,
   max_sentences: 10,
-  target_reduction: 0.45
+  target_reduction: 0.45,
+  mode: "conservative"
 };
 function tokenizeForHeuristicCompression(text) {
   if (!text) {
@@ -652,25 +659,46 @@ function compressHeuristicText(input, options = {}) {
       warnings: ["JSON-to-TOON skipped because the input is not a complete valid JSON object or array."]
     };
   }
-  const analysis = preprocessContext(inputText);
   if (inputTokens === 0) {
     return toCompressionOutput(inputText, inputTokens);
   }
+  const mode = options.mode ?? DEFAULT_OPTIONS.mode;
+  const isConservative = mode === "conservative";
+  const failOnLowConfidence = options.fail_on_low_confidence ?? isConservative;
+  const preserveNeighbors = options.preserve_neighbors ?? isConservative;
   const minTokens = options.min_tokens ?? DEFAULT_OPTIONS.min_tokens;
   if (inputTokens <= minTokens) {
     return toCompressionOutput(inputText, inputTokens);
   }
+  if (isConservative && looksStructured(inputText)) {
+    return toCompressionOutput(inputText, inputTokens, [
+      "Heuristic compression skipped because the input appears to contain structured blocks; use TOON or code compaction for structured content."
+    ]);
+  }
+  const analysis = preprocessContext(inputText);
   const targetReduction = options.target_reduction ?? DEFAULT_OPTIONS.target_reduction;
   const targetTokens = Math.max(1, Math.max(minTokens, Math.floor(inputTokens * (1 - targetReduction))));
   const maxSentences = options.max_sentences ?? DEFAULT_OPTIONS.max_sentences;
   const query = options.query ?? "";
   const terms = queryTerms(query);
+  if (failOnLowConfidence && terms.size === 0) {
+    return toCompressionOutput(inputText, inputTokens, [
+      "Heuristic compression skipped because no usable relevance query terms were available."
+    ]);
+  }
   const scoredSentences = analysis.candidates.map((candidate) => ({
     sentence: candidate.sentence,
     index: candidate.index,
     ...scoreSentenceCandidate(candidate, terms),
-    tokens: candidate.tokenCount
+    tokens: candidate.tokenCount,
+    isProtected: candidate.isProtected,
+    hasEvidence: candidate.hasEvidence
   }));
+  if (failOnLowConfidence && terms.size > 0 && !scoredSentences.some((candidate) => candidate.overlapCount > 0)) {
+    return toCompressionOutput(inputText, inputTokens, [
+      "Heuristic compression skipped because no sentence matched the relevance query."
+    ]);
+  }
   const selected = [];
   const selectedIndices = /* @__PURE__ */ new Set();
   const coveredTerms = /* @__PURE__ */ new Set();
@@ -710,19 +738,15 @@ function compressHeuristicText(input, options = {}) {
       coveredTerms.add(term);
     }
   }
-  selected.sort((left, right) => left.index - right.index);
-  let output = selected.map((item) => item.sentence).join(" ");
+  const finalSelection = preserveNeighbors ? expandSelectionWithNeighbors(selected, scoredSentences, maxSentences) : selected;
+  finalSelection.sort((left, right) => left.index - right.index);
+  let output = finalSelection.map((item) => item.sentence).join(" ");
   if (!output) {
-    output = tokenizeForHeuristicCompression(inputText).slice(0, targetTokens).join(" ");
-  } else {
-    const outputTokens = tokenizeForHeuristicCompression(output);
-    if (outputTokens.length > targetTokens) {
-      output = outputTokens.slice(0, targetTokens).join(" ");
-    }
+    output = takeWholeCandidatesWithinBudget(analysis.candidates, targetTokens) || inputText;
   }
   return toCompressionOutput(output, inputTokens);
 }
-function toCompressionOutput(output, inputTokens) {
+function toCompressionOutput(output, inputTokens, warnings = []) {
   const outputTokens = estimateHeuristicTokens(output);
   const tokensSaved = Math.max(0, inputTokens - outputTokens);
   return {
@@ -730,7 +754,8 @@ function toCompressionOutput(output, inputTokens) {
     inputTokens,
     outputTokens,
     tokensSaved,
-    compressionRatio: outputTokens === 0 ? 0 : inputTokens / outputTokens
+    compressionRatio: outputTokens === 0 ? 0 : inputTokens / outputTokens,
+    ...warnings.length > 0 ? { warnings } : {}
   };
 }
 function isBetterTieBreak(candidate, bestCandidate) {
@@ -748,13 +773,6 @@ function normalizeForDeduping(sentence) {
 function splitIntoSentences(text) {
   return text.split(/(?<=[.!?])\s+|\n+/g).map((sentence) => sentence.trim()).filter(Boolean);
 }
-function chunkTokens(tokens, chunkSize) {
-  const chunks = [];
-  for (let index = 0; index < tokens.length; index += chunkSize) {
-    chunks.push(tokens.slice(index, index + chunkSize));
-  }
-  return chunks;
-}
 function splitOversizedSentence(sentence, maxTokens = MAX_SEGMENT_TOKENS) {
   const rawTokens = tokenizeForHeuristicCompression(sentence);
   if (rawTokens.length <= maxTokens) {
@@ -766,10 +784,10 @@ function splitOversizedSentence(sentence, maxTokens = MAX_SEGMENT_TOKENS) {
       if (estimateHeuristicTokens(segment) <= maxTokens) {
         return [segment];
       }
-      return chunkTokens(tokenizeForHeuristicCompression(segment), maxTokens).map((chunk) => chunk.join(" "));
+      return [segment];
     });
   }
-  return chunkTokens(rawTokens, maxTokens).map((chunk) => chunk.join(" "));
+  return [sentence];
 }
 function normalizeToken(token) {
   return token.replace(TOKEN_NORMALIZE_REGEX, "").trim();
@@ -778,19 +796,24 @@ function tokenizeNormalized(text) {
   return tokenizeForHeuristicCompression(text.toLowerCase()).map(normalizeToken).filter(Boolean);
 }
 function queryTerms(query) {
-  return new Set(tokenizeNormalized(query).filter((term) => term.length >= 3 && !STOP_WORDS.has(term)));
+  return new Set(tokenizeNormalized(query).filter((term) => {
+    if (STOP_WORDS.has(term)) {
+      return false;
+    }
+    return term.length >= 3 || /^(?:[a-z]\d|\d+[a-z]?|[a-z]{2})$/i.test(term);
+  }));
 }
 function preprocessContext(context) {
   const originalTokens = estimateHeuristicTokens(context);
-  const dedupeSet = /* @__PURE__ */ new Set();
   const candidates = [];
+  let previousNormalized = "";
   for (const sentence of splitIntoSentences(context)) {
     for (const segment of splitOversizedSentence(sentence)) {
       const normalized = normalizeForDeduping(segment);
-      if (!normalized || dedupeSet.has(normalized)) {
+      if (!normalized || normalized === previousNormalized) {
         continue;
       }
-      dedupeSet.add(normalized);
+      previousNormalized = normalized;
       const rawTokens = tokenizeForHeuristicCompression(segment);
       const normalizedTerms = rawTokens.map((token) => normalizeToken(token.toLowerCase())).filter(Boolean);
       const lowered = segment.toLowerCase();
@@ -799,7 +822,9 @@ function preprocessContext(context) {
         index: candidates.length,
         tokenCount: rawTokens.length,
         uniqueTerms: new Set(normalizedTerms),
-        isBoilerplate: BOILERPLATE_REGEX.test(lowered)
+        isBoilerplate: BOILERPLATE_REGEX.test(lowered),
+        isProtected: PROTECTED_SIGNAL_REGEX.test(segment),
+        hasEvidence: EVIDENCE_SIGNAL_REGEX.test(segment)
       });
     }
   }
@@ -825,12 +850,62 @@ function scoreSentenceCandidate(candidate, terms) {
   const densityScore = terms.size > 0 ? overlap / terms.size : 0;
   const lengthScore = Math.min(candidate.tokenCount, 40) / 40;
   const boilerplatePenalty = candidate.isBoilerplate ? 0.5 : 0;
+  const protectedScore = candidate.isProtected ? 0.9 : 0;
+  const evidenceScore = candidate.hasEvidence ? 0.4 : 0;
   return {
-    score: overlapScore + densityScore + lengthScore - boilerplatePenalty,
+    score: overlapScore + densityScore + lengthScore + protectedScore + evidenceScore - boilerplatePenalty,
     overlapCount: overlap,
     overlapTerms
   };
 }
+function looksStructured(text) {
+  if (STRUCTURED_BLOCK_REGEX.test(text)) {
+    return true;
+  }
+  const lines = text.split(/\r?\n/).map((line2) => line2.trim()).filter(Boolean);
+  if (lines.length < 3) {
+    return false;
+  }
+  const listLikeLines = lines.filter((line2) => /^[-*+] |\d+[.)] /.test(line2)).length;
+  return listLikeLines >= 3 && listLikeLines / lines.length >= 0.6;
+}
+function expandSelectionWithNeighbors(selected, scoredSentences, maxSentences) {
+  if (selected.length === 0 || selected.length >= maxSentences) {
+    return selected;
+  }
+  const byIndex = new Map(scoredSentences.map((candidate) => [candidate.index, candidate]));
+  const expanded = new Map(selected.map((candidate) => [candidate.index, candidate]));
+  for (const candidate of selected) {
+    if (expanded.size >= maxSentences) {
+      break;
+    }
+    if (candidate.overlapCount === 0 && !candidate.isProtected && !candidate.hasEvidence) {
+      continue;
+    }
+    for (const neighborIndex of [candidate.index - 1, candidate.index + 1]) {
+      if (expanded.size >= maxSentences) {
+        break;
+      }
+      const neighbor = byIndex.get(neighborIndex);
+      if (neighbor && !expanded.has(neighbor.index)) {
+        expanded.set(neighbor.index, neighbor);
+      }
+    }
+  }
+  return [...expanded.values()];
+}
+function takeWholeCandidatesWithinBudget(candidates, targetTokens) {
+  const selected = [];
+  let tokens = 0;
+  for (const candidate of candidates) {
+    if (tokens + candidate.tokenCount > targetTokens) {
+      break;
+    }
+    selected.push(candidate.sentence);
+    tokens += candidate.tokenCount;
+  }
+  return selected.join(" ");
+}
 
 // src/code-compaction.ts
 function compactCode(input, options = {}) {
@@ -996,7 +1071,10 @@ function renderSections(asset, options = {}) {
         max_sentences: configuredHeuristic?.max_sentences,
         target_reduction: configuredHeuristic?.target_reduction,
         query,
-        json_to_toon: configuredHeuristic?.json_to_toon
+        json_to_toon: configuredHeuristic?.json_to_toon,
+        mode: configuredHeuristic?.mode,
+        preserve_neighbors: configuredHeuristic?.preserve_neighbors,
+        fail_on_low_confidence: configuredHeuristic?.fail_on_low_confidence
       };
       const cacheKey = JSON.stringify([name, value, compressionOptions]);
       const cached = compressionCache.get(cacheKey);
@@ -1520,7 +1598,10 @@ async function applyPromptCompressionForRender(asset, runtime) {
       max_sentences: heuristicConfig?.max_sentences,
       target_reduction: heuristicConfig?.target_reduction,
       query: resolveHeuristicPromptQuery(heuristicConfig, runtime.variables, sections.system_instructions),
-      json_to_toon: heuristicConfig?.json_to_toon
+      json_to_toon: heuristicConfig?.json_to_toon,
+      mode: heuristicConfig?.mode,
+      preserve_neighbors: heuristicConfig?.preserve_neighbors,
+      fail_on_low_confidence: heuristicConfig?.fail_on_low_confidence
     });
     promptTemplate = result.output;
     reportHeuristicCompressionWarnings2(warnings, result.warnings, "prompt template");
@@ -1757,4 +1838,4 @@ export {
   withPromptInputSupport,
   applyRawProviderBody
 };
-//# sourceMappingURL=chunk-RVAFVWR2.js.map
+//# sourceMappingURL=chunk-NS6OTKY2.js.map