promptopskit 0.8.2 → 0.9.1

package/dist/index.cjs

@@ -153,7 +153,10 @@ function normalizeContextInputCompression(value) {
       target_reduction: value.heuristic.target_reduction,
       query: value.heuristic.query,
       query_variable: value.heuristic.query_variable,
-      json_to_toon: value.heuristic.json_to_toon
+      json_to_toon: value.heuristic.json_to_toon,
+      mode: value.heuristic.mode,
+      preserve_neighbors: value.heuristic.preserve_neighbors,
+      fail_on_low_confidence: value.heuristic.fail_on_low_confidence
     }
   };
   const code = normalizeCodeCompression(value.code);
@@ -658,7 +661,10 @@ function line(depth, content, indent) {
 var TOKEN_REGEX = /[\p{L}\p{N}]+|[^\s]/gu;
 var TOKEN_NORMALIZE_REGEX = /[^\p{L}\p{N}]/gu;
 var DEDUPE_NORMALIZE_REGEX = /[^\p{L}\p{N}\s]/gu;
-var BOILERPLATE_REGEX = /copyright|all rights reserved|disclaimer|confidential/i;
+var BOILERPLATE_REGEX = /copyright|all rights reserved|disclaimer/i;
+var STRUCTURED_BLOCK_REGEX = /(^|\n)\s*```|(^|\n)\s*\|.+\|\s*(\n|$)/;
+var PROTECTED_SIGNAL_REGEX = /\b(?:must|shall|should|required|requires|requirement|never|do not|don't|only|except|unless|however|but|constraint|constraints|safety|security|deadline|sla)\b|output\s+format/i;
+var EVIDENCE_SIGNAL_REGEX = /https?:\/\/|\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b|[$€£]\s?\d|\b\d+(?:[.,:/-]\d+)*%?\b|\b[A-Z]{2,}[-_A-Z0-9]*\b|\b[A-Z]\d\b/i;
 var MAX_SEGMENT_TOKENS = 120;
 var STOP_WORDS = /* @__PURE__ */ new Set([
   "a",
@@ -692,7 +698,8 @@ var STOP_WORDS = /* @__PURE__ */ new Set([
 var DEFAULT_OPTIONS = {
   min_tokens: 80,
   max_sentences: 10,
-  target_reduction: 0.45
+  target_reduction: 0.45,
+  mode: "conservative"
 };
 function tokenizeForHeuristicCompression(text) {
   if (!text) {
@@ -726,25 +733,46 @@ function compressHeuristicText(input, options = {}) {
       warnings: ["JSON-to-TOON skipped because the input is not a complete valid JSON object or array."]
     };
   }
-  const analysis = preprocessContext(inputText);
   if (inputTokens === 0) {
     return toCompressionOutput(inputText, inputTokens);
   }
+  const mode = options.mode ?? DEFAULT_OPTIONS.mode;
+  const isConservative = mode === "conservative";
+  const failOnLowConfidence = options.fail_on_low_confidence ?? isConservative;
+  const preserveNeighbors = options.preserve_neighbors ?? isConservative;
   const minTokens = options.min_tokens ?? DEFAULT_OPTIONS.min_tokens;
   if (inputTokens <= minTokens) {
     return toCompressionOutput(inputText, inputTokens);
   }
+  if (isConservative && looksStructured(inputText)) {
+    return toCompressionOutput(inputText, inputTokens, [
+      "Heuristic compression skipped because the input appears to contain structured blocks; use TOON or code compaction for structured content."
+    ]);
+  }
+  const analysis = preprocessContext(inputText);
   const targetReduction = options.target_reduction ?? DEFAULT_OPTIONS.target_reduction;
   const targetTokens = Math.max(1, Math.max(minTokens, Math.floor(inputTokens * (1 - targetReduction))));
   const maxSentences = options.max_sentences ?? DEFAULT_OPTIONS.max_sentences;
   const query = options.query ?? "";
   const terms = queryTerms(query);
+  if (failOnLowConfidence && terms.size === 0) {
+    return toCompressionOutput(inputText, inputTokens, [
+      "Heuristic compression skipped because no usable relevance query terms were available."
+    ]);
+  }
   const scoredSentences = analysis.candidates.map((candidate) => ({
     sentence: candidate.sentence,
     index: candidate.index,
     ...scoreSentenceCandidate(candidate, terms),
-    tokens: candidate.tokenCount
+    tokens: candidate.tokenCount,
+    isProtected: candidate.isProtected,
+    hasEvidence: candidate.hasEvidence
   }));
+  if (failOnLowConfidence && terms.size > 0 && !scoredSentences.some((candidate) => candidate.overlapCount > 0)) {
+    return toCompressionOutput(inputText, inputTokens, [
+      "Heuristic compression skipped because no sentence matched the relevance query."
+    ]);
+  }
   const selected = [];
   const selectedIndices = /* @__PURE__ */ new Set();
   const coveredTerms = /* @__PURE__ */ new Set();
@@ -784,19 +812,15 @@ function compressHeuristicText(input, options = {}) {
       coveredTerms.add(term);
     }
   }
-  selected.sort((left, right) => left.index - right.index);
-  let output = selected.map((item) => item.sentence).join(" ");
+  const finalSelection = preserveNeighbors ? expandSelectionWithNeighbors(selected, scoredSentences, maxSentences) : selected;
+  finalSelection.sort((left, right) => left.index - right.index);
+  let output = finalSelection.map((item) => item.sentence).join(" ");
   if (!output) {
-    output = tokenizeForHeuristicCompression(inputText).slice(0, targetTokens).join(" ");
-  } else {
-    const outputTokens = tokenizeForHeuristicCompression(output);
-    if (outputTokens.length > targetTokens) {
-      output = outputTokens.slice(0, targetTokens).join(" ");
-    }
+    output = takeWholeCandidatesWithinBudget(analysis.candidates, targetTokens) || inputText;
   }
   return toCompressionOutput(output, inputTokens);
 }
-function toCompressionOutput(output, inputTokens) {
+function toCompressionOutput(output, inputTokens, warnings = []) {
   const outputTokens = estimateHeuristicTokens(output);
   const tokensSaved = Math.max(0, inputTokens - outputTokens);
   return {
@@ -804,7 +828,8 @@ function toCompressionOutput(output, inputTokens) {
     inputTokens,
     outputTokens,
     tokensSaved,
-    compressionRatio: outputTokens === 0 ? 0 : inputTokens / outputTokens
+    compressionRatio: outputTokens === 0 ? 0 : inputTokens / outputTokens,
+    ...warnings.length > 0 ? { warnings } : {}
   };
 }
 function isBetterTieBreak(candidate, bestCandidate) {
@@ -822,13 +847,6 @@ function normalizeForDeduping(sentence) {
 function splitIntoSentences(text) {
   return text.split(/(?<=[.!?])\s+|\n+/g).map((sentence) => sentence.trim()).filter(Boolean);
 }
-function chunkTokens(tokens, chunkSize) {
-  const chunks = [];
-  for (let index = 0; index < tokens.length; index += chunkSize) {
-    chunks.push(tokens.slice(index, index + chunkSize));
-  }
-  return chunks;
-}
 function splitOversizedSentence(sentence, maxTokens = MAX_SEGMENT_TOKENS) {
   const rawTokens = tokenizeForHeuristicCompression(sentence);
   if (rawTokens.length <= maxTokens) {
@@ -840,10 +858,10 @@ function splitOversizedSentence(sentence, maxTokens = MAX_SEGMENT_TOKENS) {
       if (estimateHeuristicTokens(segment) <= maxTokens) {
         return [segment];
       }
-      return chunkTokens(tokenizeForHeuristicCompression(segment), maxTokens).map((chunk) => chunk.join(" "));
+      return [segment];
     });
   }
-  return chunkTokens(rawTokens, maxTokens).map((chunk) => chunk.join(" "));
+  return [sentence];
 }
 function normalizeToken(token) {
   return token.replace(TOKEN_NORMALIZE_REGEX, "").trim();
@@ -852,19 +870,24 @@ function tokenizeNormalized(text) {
   return tokenizeForHeuristicCompression(text.toLowerCase()).map(normalizeToken).filter(Boolean);
 }
 function queryTerms(query) {
-  return new Set(tokenizeNormalized(query).filter((term) => term.length >= 3 && !STOP_WORDS.has(term)));
+  return new Set(tokenizeNormalized(query).filter((term) => {
+    if (STOP_WORDS.has(term)) {
+      return false;
+    }
+    return term.length >= 3 || /^(?:[a-z]\d|\d+[a-z]?|[a-z]{2})$/i.test(term);
+  }));
 }
 function preprocessContext(context) {
   const originalTokens = estimateHeuristicTokens(context);
-  const dedupeSet = /* @__PURE__ */ new Set();
   const candidates = [];
+  let previousNormalized = "";
   for (const sentence of splitIntoSentences(context)) {
     for (const segment of splitOversizedSentence(sentence)) {
       const normalized = normalizeForDeduping(segment);
-      if (!normalized || dedupeSet.has(normalized)) {
+      if (!normalized || normalized === previousNormalized) {
         continue;
       }
-      dedupeSet.add(normalized);
+      previousNormalized = normalized;
       const rawTokens = tokenizeForHeuristicCompression(segment);
       const normalizedTerms = rawTokens.map((token) => normalizeToken(token.toLowerCase())).filter(Boolean);
       const lowered = segment.toLowerCase();
@@ -873,7 +896,9 @@ function preprocessContext(context) {
         index: candidates.length,
         tokenCount: rawTokens.length,
         uniqueTerms: new Set(normalizedTerms),
-        isBoilerplate: BOILERPLATE_REGEX.test(lowered)
+        isBoilerplate: BOILERPLATE_REGEX.test(lowered),
+        isProtected: PROTECTED_SIGNAL_REGEX.test(segment),
+        hasEvidence: EVIDENCE_SIGNAL_REGEX.test(segment)
       });
     }
   }
@@ -899,12 +924,62 @@ function scoreSentenceCandidate(candidate, terms) {
   const densityScore = terms.size > 0 ? overlap / terms.size : 0;
   const lengthScore = Math.min(candidate.tokenCount, 40) / 40;
   const boilerplatePenalty = candidate.isBoilerplate ? 0.5 : 0;
+  const protectedScore = candidate.isProtected ? 0.9 : 0;
+  const evidenceScore = candidate.hasEvidence ? 0.4 : 0;
   return {
-    score: overlapScore + densityScore + lengthScore - boilerplatePenalty,
+    score: overlapScore + densityScore + lengthScore + protectedScore + evidenceScore - boilerplatePenalty,
     overlapCount: overlap,
     overlapTerms
   };
 }
+function looksStructured(text) {
+  if (STRUCTURED_BLOCK_REGEX.test(text)) {
+    return true;
+  }
+  const lines = text.split(/\r?\n/).map((line2) => line2.trim()).filter(Boolean);
+  if (lines.length < 3) {
+    return false;
+  }
+  const listLikeLines = lines.filter((line2) => /^[-*+] |\d+[.)] /.test(line2)).length;
+  return listLikeLines >= 3 && listLikeLines / lines.length >= 0.6;
+}
+function expandSelectionWithNeighbors(selected, scoredSentences, maxSentences) {
+  if (selected.length === 0 || selected.length >= maxSentences) {
+    return selected;
+  }
+  const byIndex = new Map(scoredSentences.map((candidate) => [candidate.index, candidate]));
+  const expanded = new Map(selected.map((candidate) => [candidate.index, candidate]));
+  for (const candidate of selected) {
+    if (expanded.size >= maxSentences) {
+      break;
+    }
+    if (candidate.overlapCount === 0 && !candidate.isProtected && !candidate.hasEvidence) {
+      continue;
+    }
+    for (const neighborIndex of [candidate.index - 1, candidate.index + 1]) {
+      if (expanded.size >= maxSentences) {
+        break;
+      }
+      const neighbor = byIndex.get(neighborIndex);
+      if (neighbor && !expanded.has(neighbor.index)) {
+        expanded.set(neighbor.index, neighbor);
+      }
+    }
+  }
+  return [...expanded.values()];
+}
+function takeWholeCandidatesWithinBudget(candidates, targetTokens) {
+  const selected = [];
+  let tokens = 0;
+  for (const candidate of candidates) {
+    if (tokens + candidate.tokenCount > targetTokens) {
+      break;
+    }
+    selected.push(candidate.sentence);
+    tokens += candidate.tokenCount;
+  }
+  return selected.join(" ");
+}
 
 // src/code-compaction.ts
 function compactCode(input, options = {}) {
@@ -1070,7 +1145,10 @@ function renderSections(asset, options = {}) {
         max_sentences: configuredHeuristic?.max_sentences,
         target_reduction: configuredHeuristic?.target_reduction,
         query,
-        json_to_toon: configuredHeuristic?.json_to_toon
+        json_to_toon: configuredHeuristic?.json_to_toon,
+        mode: configuredHeuristic?.mode,
+        preserve_neighbors: configuredHeuristic?.preserve_neighbors,
+        fail_on_low_confidence: configuredHeuristic?.fail_on_low_confidence
       };
       const cacheKey = JSON.stringify([name, value, compressionOptions]);
       const cached = compressionCache.get(cacheKey);
@@ -1436,7 +1514,10 @@ var HeuristicCompressionSchema = import_zod.z.object({
   target_reduction: import_zod.z.number().min(0).max(1).optional(),
   query: import_zod.z.string().optional(),
   query_variable: import_zod.z.string().min(1).optional(),
-  json_to_toon: import_zod.z.boolean().optional()
+  json_to_toon: import_zod.z.boolean().optional(),
+  mode: import_zod.z.enum(["conservative", "balanced"]).optional(),
+  preserve_neighbors: import_zod.z.boolean().optional(),
+  fail_on_low_confidence: import_zod.z.boolean().optional()
 });
 var CodeCompactionSchema = import_zod.z.object({
   enabled: import_zod.z.boolean().optional(),
@@ -2421,7 +2502,10 @@ async function applyPromptCompressionForRender(asset, runtime) {
       max_sentences: heuristicConfig?.max_sentences,
       target_reduction: heuristicConfig?.target_reduction,
       query: resolveHeuristicPromptQuery(heuristicConfig, runtime.variables, sections.system_instructions),
-      json_to_toon: heuristicConfig?.json_to_toon
+      json_to_toon: heuristicConfig?.json_to_toon,
+      mode: heuristicConfig?.mode,
+      preserve_neighbors: heuristicConfig?.preserve_neighbors,
+      fail_on_low_confidence: heuristicConfig?.fail_on_low_confidence
     });
     promptTemplate = result.output;
     reportHeuristicCompressionWarnings2(warnings, result.warnings, "prompt template");
@@ -2546,10 +2630,29 @@ async function compressWithTheTokenCompany(input, options) {
     );
   }
   const data = await response.json();
-  if (typeof data.output !== "string" || typeof data.output_tokens !== "number" || typeof data.input_tokens !== "number" || typeof data.tokens_saved !== "number" || typeof data.compression_ratio !== "number") {
+  const normalized = normalizeTheTokenCompanyCompressResponse(data);
+  if (!normalized) {
     throw new Error("TheTokenCompany compression returned an invalid response payload.");
   }
-  return data;
+  return normalized;
+}
+function normalizeTheTokenCompanyCompressResponse(data) {
+  if (typeof data.output !== "string" || typeof data.output_tokens !== "number") {
+    return void 0;
+  }
+  const inputTokens = typeof data.input_tokens === "number" ? data.input_tokens : data.original_input_tokens;
+  if (typeof inputTokens !== "number") {
+    return void 0;
+  }
+  const tokensSaved = typeof data.tokens_saved === "number" ? data.tokens_saved : inputTokens - data.output_tokens;
+  const compressionRatio = typeof data.compression_ratio === "number" ? data.compression_ratio : data.output_tokens === 0 ? 0 : inputTokens / data.output_tokens;
+  return {
+    output: data.output,
+    output_tokens: data.output_tokens,
+    input_tokens: inputTokens,
+    tokens_saved: tokensSaved,
+    compression_ratio: compressionRatio
+  };
 }
 function getEnv(name) {
   if (typeof process === "undefined") {