toolnexus 0.26.0 → 0.28.0

package/src/skill.ts

@@ -12,10 +12,19 @@
 import { readFileSync, readdirSync, realpathSync, statSync } from "node:fs"
 import path from "node:path"
 import { pathToFileURL } from "node:url"
-import { parse as parseYaml } from "yaml"
+import { createRequire } from "node:module"
 import { compareCodePoints, compareDiscovery, sortEntriesByName } from "./order.js"
 import type { Tool, ToolResult } from "./types.js"
 
+// yaml is required on the first frontmatter parse, not at import, so a host with no skills never
+// loads it. `require` keeps the parse synchronous (listSkills/loadSkills are sync); yaml's Node
+// entry is CommonJS, so this is the same module instance an `import` would get.
+let yamlParse: ((src: string) => unknown) | undefined
+function parseYaml(src: string): unknown {
+  yamlParse ??= (createRequire(import.meta.url)("yaml") as typeof import("yaml")).parse
+  return yamlParse(src)
+}
+
 export const SKILL_TOOL_DESCRIPTION = `Load a specialized skill when the task at hand matches one of the skills listed in the system prompt.
 
 Use this tool to inject the skill's instructions and resources into current conversation. The output may contain detailed workflow guidance as well as references to scripts, files, etc in the same directory as the skill.
@@ -68,10 +77,25 @@ export interface SkillSkip {
   detail?: string
 }
 
+/** Why a KEPT skill carries a warning (SPEC.md §3, #84). The skill still loads unchanged. */
+export type SkillWarningReason = "oversized-description"
+
+/** Maximum skill `description` length, in Unicode code points, before a warning (SPEC.md §3). */
+export const SKILL_DESCRIPTION_MAX_CHARS = 1024
+
+export interface SkillWarning {
+  location: string
+  reason: SkillWarningReason
+  /** Description length in Unicode CODE POINTS (not bytes, not UTF-16 units). */
+  chars: number
+}
+
 /** Result of a list-only validate pass (SPEC.md §3, S3). */
 export interface SkillInventory {
   skills: SkillInfo[]
   skipped: SkillSkip[]
+  /** Kept skills that load but carry a warning (oversized description), in discovery order. */
+  warnings: SkillWarning[]
 }
 
 /** Options for loadSkills / listSkills (SPEC.md §3, S1/S2/S5). */
@@ -353,6 +377,26 @@ function collectCandidates(opts: LoadSkillsOptions): RawCandidate[] {
   return cands
 }
 
+/**
+ * Warn — never modify — skills whose description exceeds SKILL_DESCRIPTION_MAX_CHARS code
+ * points (SPEC.md §3, #84). Each warning is also sent to the same channel as the
+ * duplicate-name warning. Order follows the given skills (discovery order).
+ */
+function describeWarnings(skills: SkillInfo[]): SkillWarning[] {
+  const out: SkillWarning[] = []
+  for (const s of skills) {
+    if (s.description === undefined) continue
+    const chars = [...s.description].length
+    if (chars > SKILL_DESCRIPTION_MAX_CHARS) {
+      console.warn(
+        `[toolnexus] skill "${s.name}" (${s.location}) description is ${chars} chars, over ${SKILL_DESCRIPTION_MAX_CHARS} — kept unmodified`,
+      )
+      out.push({ location: s.location, reason: "oversized-description", chars })
+    }
+  }
+  return out
+}
+
 /** Dedupe candidates by name (first-wins); later duplicates become skips. */
 function mergeCandidates(cands: RawCandidate[]): { skills: Record<string, SkillInfo>; skipped: SkillSkip[] } {
   const skills: Record<string, SkillInfo> = {}
@@ -409,6 +453,8 @@ export interface SkillSource {
    * that stops being reported is indistinguishable from something that was finished.
    */
   skipped: SkillSkip[]
+  /** Exposed (post-allowlist) skills whose description is oversized; never modified (#84). */
+  warnings: SkillWarning[]
 }
 
 /**
@@ -420,7 +466,8 @@ export function listSkills(input: string | string[] | LoadSkillsOptions): SkillI
   const opts: LoadSkillsOptions =
     typeof input === "string" || Array.isArray(input) ? { dirs: input } : input
   const merged = mergeCandidates(collectCandidates(opts))
-  return { skills: Object.values(merged.skills), skipped: merged.skipped }
+  const list = Object.values(merged.skills)
+  return { skills: list, skipped: merged.skipped, warnings: describeWarnings(list) }
 }
 
 /** Discover skills (dirs and/or data) and build the `skill` loader tool. */
@@ -432,6 +479,7 @@ export function loadSkills(input: string | string[] | LoadSkillsOptions): SkillS
   const skills = applySkillsFilter(merged.skills, opts.filter)
 
   const skipped = merged.skipped
+  const warnings = describeWarnings(Object.values(skills))
   const tool: Tool = {
     name: "skill",
     description: SKILL_TOOL_DESCRIPTION,
@@ -499,6 +547,7 @@ export function loadSkills(input: string | string[] | LoadSkillsOptions): SkillS
     skills,
     tool,
     skipped,
+    warnings,
     prompt() {
       const described = Object.values(skills)
         .filter((s) => s.description !== undefined)

package/src/transcript_search.ts

@@ -0,0 +1,172 @@
+/**
+ * `transcriptSearchTool` — read-only search over a {@link ConversationStore} (SPEC.md §7E
+ * "Memory hierarchy" → transcript search). The ground-truth tier of agent memory: a dreaming
+ * persona checks its notes against what actually happened, not against its own notes.
+ */
+import { defineTool } from "./native.js"
+import type { Tool } from "./types.js"
+import type { ConversationEntry, ConversationStore } from "./client.js"
+
+/** Options for {@link transcriptSearchTool}. */
+export interface TranscriptSearchOptions {
+  /** Most hits returned per call (and the ceiling for the `limit` input). Default 20. */
+  maxResults?: number
+  /** Snippet length in characters (Unicode code points, not bytes). Default 200. */
+  snippetChars?: number
+  /** Tool name. Default `transcript_search`. */
+  name?: string
+}
+
+/** A store that can enumerate its conversations — what transcript search requires. */
+export type ListableConversationStore = ConversationStore & { list(): Promise<ConversationEntry[]> }
+
+export const TRANSCRIPT_SEARCH_DESCRIPTION =
+  "Search past conversation transcripts — the ground truth of what actually happened. " +
+  "Case-insensitive substring match over message text. Returns one line per matching " +
+  "message: <conversationId>#<index> <role>: <snippet>. Read-only."
+
+const WS = /[ \t\n\r\f\v]+/g
+
+/** Collapse every run of ASCII whitespace to one space and trim (§7E). */
+function normalize(s: string): string {
+  return s.replace(WS, " ").replace(/^ | $/g, "")
+}
+
+/** ASCII-only case fold — never Unicode lowercasing (it changes lengths, differs per port). */
+function fold(s: string): string {
+  return s.replace(/[A-Z]/g, (c) => String.fromCharCode(c.charCodeAt(0) + 32))
+}
+
+/** Code-point order (not UTF-16 code-unit order, which differs above U+FFFF). */
+function byCodePoint(a: string, b: string): number {
+  const x = Array.from(a)
+  const y = Array.from(b)
+  for (let i = 0; i < x.length && i < y.length; i++) {
+    const d = x[i].codePointAt(0)! - y[i].codePointAt(0)!
+    if (d !== 0) return d
+  }
+  return x.length - y.length
+}
+
+/** A copy with every object's keys sorted by code point, recursively (§7E); arrays keep order. */
+function sortKeys(v: unknown): unknown {
+  if (Array.isArray(v)) return v.map(sortKeys)
+  if (v && typeof v === "object" && Object.getPrototypeOf(v) === Object.prototype) {
+    const o = v as Record<string, unknown>
+    const out: Record<string, unknown> = {}
+    for (const k of Object.keys(o).sort(byCodePoint)) out[k] = sortKeys(o[k])
+    return out
+  }
+  return v
+}
+
+function partsText(content: unknown): string[] {
+  if (typeof content === "string") return [content]
+  if (!Array.isArray(content)) return []
+  const out: string[] = []
+  for (const p of content) {
+    if (!p || typeof p !== "object") continue
+    const part = p as Record<string, unknown>
+    if (part.type === "text" && typeof part.text === "string") out.push(part.text)
+    else if (part.type === "tool_use") out.push(JSON.stringify(sortKeys(part.input ?? {})))
+    else if (part.type === "tool_result") out.push(...partsText(part.content))
+  }
+  return out
+}
+
+/** The searchable text of one stored message (§7E): string content, text parts, tool_use
+ * input as compact JSON, tool_result content, then each OpenAI tool call's `arguments`
+ * string — pieces joined by "\n", then whitespace-normalized. Images are not text. */
+export function messageText(msg: unknown): string {
+  if (!msg || typeof msg !== "object") return ""
+  const m = msg as Record<string, unknown>
+  const pieces = partsText(m.content)
+  if (Array.isArray(m.tool_calls)) {
+    for (const tc of m.tool_calls as any[]) {
+      const a = tc?.function?.arguments
+      if (typeof a === "string") pieces.push(a)
+    }
+  }
+  return normalize(pieces.join("\n"))
+}
+
+/** Snippet of `snippetChars` code points centred on the match; `…` marks elision. */
+function snippet(cps: string[], at: number, qlen: number, size: number): string {
+  if (cps.length <= size) return cps.join("")
+  let start = at + Math.floor(qlen / 2) - Math.floor(size / 2)
+  start = Math.max(0, Math.min(start, cps.length - size))
+  const end = start + size
+  return (start > 0 ? "…" : "") + cps.slice(start, end).join("") + (end < cps.length ? "…" : "")
+}
+
+/**
+ * A read-only {@link Tool} over `store` (§7E). Construction throws unless the store has
+ * `list()`. Input `{query, conversationId?, limit?}`; output one line per hit,
+ * `<conversationId>#<index> <role>: <snippet>`, `no matches` when none, and a trailing
+ * `(+<k> more)` when hits exceed the limit. Never writes.
+ */
+export function transcriptSearchTool(store: ConversationStore, opts: TranscriptSearchOptions = {}): Tool {
+  if (!store || typeof (store as Partial<ListableConversationStore>).list !== "function") {
+    throw new Error("transcriptSearchTool: the store must implement list()")
+  }
+  const listable = store as ListableConversationStore
+  const maxResults = opts.maxResults ?? 20
+  const snippetChars = opts.snippetChars ?? 200
+  if (!Number.isInteger(maxResults) || maxResults < 1) throw new Error("transcriptSearchTool: maxResults must be an integer >= 1")
+  if (!Number.isInteger(snippetChars) || snippetChars < 1) throw new Error("transcriptSearchTool: snippetChars must be an integer >= 1")
+  return defineTool({
+    name: opts.name ?? "transcript_search",
+    description: TRANSCRIPT_SEARCH_DESCRIPTION,
+    inputSchema: {
+      type: "object",
+      properties: {
+        query: { type: "string", description: "Text to find (case-insensitive substring)." },
+        conversationId: { type: "string", description: "Search only this conversation." },
+        limit: { type: "integer", description: `Most hits to return (default and max ${maxResults}).` },
+      },
+      required: ["query"],
+    },
+    run: async (args: Record<string, unknown>) => {
+      const query = typeof args.query === "string" ? normalize(args.query) : ""
+      if (query === "") return { output: "query is required", isError: true }
+      let limit = maxResults
+      if (args.limit !== undefined) {
+        if (!Number.isInteger(args.limit) || (args.limit as number) < 1) {
+          return { output: "limit must be an integer >= 1", isError: true }
+        }
+        limit = Math.min(args.limit as number, maxResults)
+      }
+      const ids =
+        typeof args.conversationId === "string" ? [args.conversationId] : (await listable.list()).map((e) => e.id)
+      const q = Array.from(fold(query))
+      const hits: string[] = []
+      let total = 0
+      for (const id of ids) {
+        const msgs = await listable.get(id)
+        if (!msgs) continue
+        msgs.forEach((msg, index) => {
+          const cps = Array.from(messageText(msg))
+          const hay = Array.from(fold(cps.join("")))
+          const at = indexOfCps(hay, q)
+          if (at < 0) return
+          total++
+          if (hits.length < limit) {
+            const role = typeof (msg as any)?.role === "string" ? (msg as any).role : "unknown"
+            hits.push(`${id}#${index} ${role}: ${snippet(cps, at, q.length, snippetChars)}`)
+          }
+        })
+      }
+      if (total === 0) return "no matches"
+      if (total > hits.length) hits.push(`(+${total - hits.length} more)`)
+      return hits.join("\n")
+    },
+  })
+}
+
+function indexOfCps(hay: string[], needle: string[]): number {
+  outer: for (let i = 0; i + needle.length <= hay.length; i++) {
+    for (let j = 0; j < needle.length; j++) if (hay[i + j] !== needle[j]) continue outer
+    return i
+  }
+  return -1
+}