typeclaw 0.1.0

package/src/agent/tools/webfetch/strategies/snapshot.ts

@@ -0,0 +1,135 @@
+import * as cheerio from 'cheerio'
+import type { AnyNode, Element } from 'domhandler'
+
+const SEMANTIC_TAGS = new Set([
+  'header',
+  'nav',
+  'main',
+  'aside',
+  'footer',
+  'section',
+  'article',
+  'h1',
+  'h2',
+  'h3',
+  'h4',
+  'h5',
+  'h6',
+  'form',
+  'input',
+  'button',
+  'select',
+  'textarea',
+  'label',
+  'a',
+  'img',
+  'ul',
+  'ol',
+  'li',
+  'table',
+  'thead',
+  'tbody',
+  'tr',
+  'th',
+  'td',
+])
+
+const ROLE_FOR_TAG: Record<string, string> = {
+  h1: 'heading',
+  h2: 'heading',
+  h3: 'heading',
+  h4: 'heading',
+  h5: 'heading',
+  h6: 'heading',
+  a: 'link',
+  button: 'button',
+  input: 'input',
+  select: 'select',
+  textarea: 'textarea',
+  img: 'image',
+  form: 'form',
+  nav: 'navigation',
+  header: 'banner',
+  footer: 'contentinfo',
+  main: 'main',
+  aside: 'complementary',
+  section: 'section',
+  article: 'article',
+  ul: 'list',
+  ol: 'list',
+  li: 'listitem',
+  table: 'table',
+  tr: 'row',
+  th: 'columnheader',
+  td: 'cell',
+  label: 'label',
+}
+
+export function applySnapshot(html: string): string {
+  const $ = cheerio.load(html)
+  const lines: string[] = []
+  const body = $('body').get(0)
+  const roots: AnyNode[] = body ? [body] : ($.root().get(0)?.children ?? [])
+  for (const root of roots) {
+    walk($, root, 0, lines)
+  }
+  return lines.length > 0 ? lines.join('\n') : 'Page contains no semantic structure.'
+}
+
+function isElement(node: AnyNode): node is Element {
+  return node.type === 'tag'
+}
+
+function walk($: cheerio.CheerioAPI, node: AnyNode, depth: number, out: string[]): void {
+  if (!isElement(node)) return
+
+  const tag = node.name.toLowerCase()
+  let nextDepth = depth
+
+  if (SEMANTIC_TAGS.has(tag)) {
+    const role = ROLE_FOR_TAG[tag] ?? tag
+    const label = labelFor($, node)
+    const indent = '  '.repeat(depth)
+    out.push(label ? `${indent}- ${role}: ${label}` : `${indent}- ${role}`)
+    nextDepth = depth + 1
+  }
+
+  for (const child of node.children) {
+    walk($, child, nextDepth, out)
+  }
+}
+
+function labelFor($: cheerio.CheerioAPI, element: Element): string {
+  const $el = $(element)
+  const tag = element.name.toLowerCase()
+
+  if (tag === 'a') {
+    const text = $el.text().replace(/\s+/g, ' ').trim()
+    const href = $el.attr('href') ?? ''
+    return text && href ? `"${truncate(text, 80)}" → ${href}` : text || href
+  }
+  if (tag === 'img') {
+    const alt = $el.attr('alt') ?? ''
+    const src = $el.attr('src') ?? ''
+    return alt ? `"${truncate(alt, 80)}" (${src})` : src
+  }
+  if (tag === 'input' || tag === 'select' || tag === 'textarea') {
+    const name = $el.attr('name') ?? ''
+    const type = $el.attr('type') ?? tag
+    const placeholder = $el.attr('placeholder') ?? ''
+    const parts = [
+      type ? `type=${type}` : '',
+      name ? `name=${name}` : '',
+      placeholder ? `placeholder="${truncate(placeholder, 40)}"` : '',
+    ]
+    return parts.filter(Boolean).join(' ')
+  }
+  if (tag === 'button' || tag === 'label' || /^h[1-6]$/.test(tag) || tag === 'th' || tag === 'td') {
+    return truncate($el.text().replace(/\s+/g, ' ').trim(), 120)
+  }
+  return ''
+}
+
+function truncate(text: string, max: number): string {
+  return text.length > max ? `${text.slice(0, max - 1)}…` : text
+}

package/src/agent/tools/webfetch/tool.ts

@@ -0,0 +1,281 @@
+import { Type } from '@mariozechner/pi-ai'
+import { defineTool } from '@mariozechner/pi-coding-agent'
+
+import { fetchWithLimits, normalizeUrl, parseMimeType, WebfetchError } from './fetch'
+import { applyGrep, GrepError } from './strategies/grep'
+import { applyJq, JqError } from './strategies/jq'
+import { applyRaw } from './strategies/raw'
+import { applyReadability } from './strategies/readability'
+import { applySelector, SelectorError } from './strategies/selector'
+import { applySnapshot } from './strategies/snapshot'
+import {
+  type CompactionStrategy,
+  DEFAULT_TIMEOUT_SECONDS,
+  MAX_TIMEOUT_SECONDS,
+  OUTPUT_CAPS,
+  type WebfetchDetails,
+} from './types'
+
+const STRATEGY_VALUES = ['readability', 'jq', 'selector', 'grep', 'snapshot', 'raw'] as const
+
+export const webfetchTool = defineTool({
+  name: 'webfetch',
+  label: 'Web Fetch',
+  description:
+    'Fetch a single HTTP(S) URL and return the body, optionally compacted by a strategy. ' +
+    'Use this when the user references a specific URL or when websearch surfaced a result you need to read in full. ' +
+    'Strategy guide:\n' +
+    '- "readability": extract article content as markdown (blogs, docs, news). Default for HTML.\n' +
+    '- "jq": query JSON APIs (npm registry, GitHub API). Pass `query` (e.g. ".items[].name").\n' +
+    '- "selector": extract text from elements matching a CSS selector. Pass `selector` (e.g. ".price").\n' +
+    '- "grep": filter lines by regex with optional `before`/`after` context. Pass `pattern`.\n' +
+    '- "snapshot": indented semantic tree of the page (forms, headings, links).\n' +
+    '- "raw": no processing.\n' +
+    'If `strategy` is omitted, it is inferred from content-type. JSON responses require explicit `strategy: "jq"` (or "raw"). ' +
+    'No SSRF protection is applied; do not use on untrusted user-supplied URLs without an outer guard.',
+  parameters: Type.Object({
+    url: Type.String({
+      description: 'URL to fetch (http:// or https://). Bare hostnames are rewritten to https://.',
+    }),
+    strategy: Type.Optional(
+      Type.Union(
+        STRATEGY_VALUES.map((value) => Type.Literal(value)),
+        { description: 'How to compact the response. If omitted, auto-detected from content-type.' },
+      ),
+    ),
+    query: Type.Optional(Type.String({ description: 'jq query (required when strategy="jq")' })),
+    selector: Type.Optional(Type.String({ description: 'CSS selector (required when strategy="selector")' })),
+    pattern: Type.Optional(Type.String({ description: 'Regex pattern (required when strategy="grep")' })),
+    before: Type.Optional(Type.Integer({ description: 'grep -B context lines', minimum: 0 })),
+    after: Type.Optional(Type.Integer({ description: 'grep -A context lines', minimum: 0 })),
+    limit: Type.Optional(Type.Integer({ description: 'grep: max result lines (default 100)', minimum: 1 })),
+    offset: Type.Optional(Type.Integer({ description: 'grep: pagination offset', minimum: 0 })),
+    timeout: Type.Optional(
+      Type.Integer({
+        description: `Timeout in seconds (default ${DEFAULT_TIMEOUT_SECONDS}, max ${MAX_TIMEOUT_SECONDS}).`,
+        minimum: 1,
+        maximum: MAX_TIMEOUT_SECONDS,
+      }),
+    ),
+  }),
+
+  async execute(_toolCallId, params, signal) {
+    const startedAt = Date.now()
+    const inputUrl = params.url
+
+    let normalizedUrl: string
+    try {
+      normalizedUrl = normalizeUrl(inputUrl)
+    } catch (error) {
+      const message = error instanceof WebfetchError ? error.message : `Invalid URL: ${error}`
+      return errorResult(inputUrl, message, { startedAt })
+    }
+
+    const timeout = clampTimeout(params.timeout)
+
+    let response
+    try {
+      response = await fetchWithLimits(normalizedUrl, timeout, signal)
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error)
+      return errorResult(normalizedUrl, message, { startedAt, finalUrl: normalizedUrl })
+    }
+
+    const mime = parseMimeType(response.contentType)
+    const resolved = resolveStrategy(params.strategy, mime)
+    if (resolved.kind === 'error') {
+      return errorResult(normalizedUrl, resolved.message, {
+        startedAt,
+        finalUrl: response.finalUrl,
+        contentType: response.contentType,
+        httpStatus: response.httpStatus,
+        bytesIn: response.bytesIn,
+      })
+    }
+    const strategy = resolved.strategy
+    const autoDetected = resolved.autoDetected
+
+    const validation = validateStrategyArgs(strategy, params)
+    if (validation) {
+      return errorResult(normalizedUrl, validation, {
+        startedAt,
+        finalUrl: response.finalUrl,
+        contentType: response.contentType,
+        httpStatus: response.httpStatus,
+        bytesIn: response.bytesIn,
+        strategy,
+        autoDetected,
+      })
+    }
+
+    let output: string
+    try {
+      output = await runStrategy(strategy, response.body, response.finalUrl, params)
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error)
+      return errorResult(normalizedUrl, message, {
+        startedAt,
+        finalUrl: response.finalUrl,
+        contentType: response.contentType,
+        httpStatus: response.httpStatus,
+        bytesIn: response.bytesIn,
+        strategy,
+        autoDetected,
+      })
+    }
+
+    const capped = capOutput(output, strategy)
+    const details: WebfetchDetails = {
+      url: normalizedUrl,
+      finalUrl: response.finalUrl,
+      strategy,
+      autoDetected,
+      contentType: response.contentType,
+      httpStatus: response.httpStatus,
+      bytesIn: response.bytesIn,
+      bytesOut: byteLength(capped.text),
+      truncated: capped.truncated,
+      durationMs: Date.now() - startedAt,
+    }
+
+    return {
+      content: [{ type: 'text' as const, text: capped.text }],
+      details,
+    }
+  },
+})
+
+type WebfetchParams = {
+  url: string
+  strategy?: CompactionStrategy
+  query?: string
+  selector?: string
+  pattern?: string
+  before?: number
+  after?: number
+  limit?: number
+  offset?: number
+  timeout?: number
+}
+
+function clampTimeout(value: number | undefined): number {
+  if (value === undefined) return DEFAULT_TIMEOUT_SECONDS
+  return Math.min(Math.max(1, Math.floor(value)), MAX_TIMEOUT_SECONDS)
+}
+
+type ResolvedStrategy =
+  | { kind: 'ok'; strategy: CompactionStrategy; autoDetected: boolean }
+  | { kind: 'error'; message: string }
+
+function resolveStrategy(explicit: CompactionStrategy | undefined, mime: string): ResolvedStrategy {
+  if (explicit) return { kind: 'ok', strategy: explicit, autoDetected: false }
+
+  if (mime === 'text/html' || mime === 'application/xhtml+xml') {
+    return { kind: 'ok', strategy: 'readability', autoDetected: true }
+  }
+  if (mime === 'application/json' || mime.endsWith('+json')) {
+    return {
+      kind: 'error',
+      message: 'JSON response — pass `strategy: "jq"` with a `query`, or `strategy: "raw"` to get it untransformed.',
+    }
+  }
+  return { kind: 'ok', strategy: 'raw', autoDetected: true }
+}
+
+function validateStrategyArgs(strategy: CompactionStrategy, params: WebfetchParams): string | null {
+  if (strategy === 'jq' && !params.query) return 'Missing required arg `query` for strategy "jq".'
+  if (strategy === 'selector' && !params.selector) return 'Missing required arg `selector` for strategy "selector".'
+  if (strategy === 'grep' && !params.pattern) return 'Missing required arg `pattern` for strategy "grep".'
+  return null
+}
+
+async function runStrategy(
+  strategy: CompactionStrategy,
+  body: string,
+  url: string,
+  params: WebfetchParams,
+): Promise<string> {
+  switch (strategy) {
+    case 'raw':
+      return applyRaw(body)
+    case 'readability':
+      return applyReadability(body, url)
+    case 'jq':
+      try {
+        return await applyJq(body, params.query ?? '')
+      } catch (error) {
+        if (error instanceof JqError) throw new Error(error.message)
+        throw error
+      }
+    case 'selector':
+      try {
+        return applySelector(body, params.selector ?? '')
+      } catch (error) {
+        if (error instanceof SelectorError) throw new Error(error.message)
+        throw error
+      }
+    case 'grep':
+      try {
+        return applyGrep(body, {
+          pattern: params.pattern ?? '',
+          before: params.before,
+          after: params.after,
+          limit: params.limit,
+          offset: params.offset,
+        })
+      } catch (error) {
+        if (error instanceof GrepError) throw new Error(error.message)
+        throw error
+      }
+    case 'snapshot':
+      return applySnapshot(body)
+  }
+}
+
+function capOutput(text: string, strategy: CompactionStrategy): { text: string; truncated: boolean } {
+  const cap = OUTPUT_CAPS[strategy]
+  if (byteLength(text) <= cap) return { text, truncated: false }
+  const head = sliceByBytes(text, cap)
+  const fullKb = (byteLength(text) / 1024).toFixed(1)
+  const shownKb = (byteLength(head) / 1024).toFixed(1)
+  const footer = `\n\n[Output truncated: shown ${shownKb} KB of ${fullKb} KB. Use a more specific strategy or a tighter pattern.]`
+  return { text: `${head}${footer}`, truncated: true }
+}
+
+function errorResult(
+  url: string,
+  message: string,
+  partial: Partial<WebfetchDetails> & { startedAt: number },
+): { content: [{ type: 'text'; text: string }]; details: WebfetchDetails } {
+  const { startedAt, ...rest } = partial
+  const details: WebfetchDetails = {
+    url,
+    finalUrl: rest.finalUrl ?? url,
+    strategy: rest.strategy ?? 'none',
+    autoDetected: rest.autoDetected ?? false,
+    contentType: rest.contentType ?? '',
+    httpStatus: rest.httpStatus ?? 0,
+    bytesIn: rest.bytesIn ?? 0,
+    bytesOut: byteLength(message),
+    truncated: false,
+    durationMs: Date.now() - startedAt,
+    error: true,
+    message,
+  }
+  return {
+    content: [{ type: 'text' as const, text: message }],
+    details,
+  }
+}
+
+const ENCODER = new TextEncoder()
+
+function byteLength(text: string): number {
+  return ENCODER.encode(text).byteLength
+}
+
+function sliceByBytes(text: string, maxBytes: number): string {
+  const encoded = ENCODER.encode(text)
+  if (encoded.byteLength <= maxBytes) return text
+  return new TextDecoder('utf-8', { fatal: false }).decode(encoded.slice(0, maxBytes))
+}

package/src/agent/tools/webfetch/types.ts

@@ -0,0 +1,33 @@
+export type CompactionStrategy = 'readability' | 'jq' | 'selector' | 'grep' | 'snapshot' | 'raw'
+
+export type WebfetchDetails = {
+  url: string
+  finalUrl: string
+  strategy: CompactionStrategy | 'none'
+  autoDetected: boolean
+  contentType: string
+  httpStatus: number
+  bytesIn: number
+  bytesOut: number
+  truncated: boolean
+  durationMs: number
+  error?: boolean
+  message?: string
+}
+
+// Per-strategy output caps. Web pages are huge; aggressive caps keep the model's
+// context lean. Lifted from oh-my-openagent PR #434's lesson that the default
+// 50k-token cap is too generous for fetched content.
+export const OUTPUT_CAPS: Record<CompactionStrategy, number> = {
+  raw: 100_000,
+  jq: 50_000,
+  readability: 200_000,
+  selector: 100_000,
+  grep: 100_000,
+  snapshot: 50_000,
+}
+
+export const MAX_RESPONSE_BYTES = 5 * 1024 * 1024
+
+export const DEFAULT_TIMEOUT_SECONDS = 30
+export const MAX_TIMEOUT_SECONDS = 120

package/src/agent/tools/websearch.ts

@@ -0,0 +1,96 @@
+import { Type } from '@mariozechner/pi-ai'
+import { defineTool } from '@mariozechner/pi-coding-agent'
+
+import { ddgSearch, DdgCaptchaError, type DdgResult } from './ddg'
+import { wikipediaSearch, type WikipediaResult } from './wikipedia'
+
+const DEFAULT_LIMIT = 10
+const MAX_LIMIT = 20
+
+type WebsearchDetails = {
+  query: string
+  source: 'web' | 'wikipedia' | 'none'
+  count: number
+  results: (DdgResult | WikipediaResult)[]
+  error?: boolean
+  message?: string
+}
+
+export const websearchTool = defineTool({
+  name: 'websearch',
+  label: 'Web Search',
+  description:
+    'Search the public web. Returns a ranked list of {title, url, snippet} entries. Use `source: "wikipedia"` for encyclopedic lookups; otherwise default to general web results from DuckDuckGo. Pair this with the `read` tool by visiting URLs you find with `bash` (curl) when you need full page contents.',
+  parameters: Type.Object({
+    query: Type.String({ description: 'The search query.' }),
+    limit: Type.Optional(
+      Type.Integer({
+        description: `Max number of results to return (1-${MAX_LIMIT}, default ${DEFAULT_LIMIT}).`,
+        minimum: 1,
+        maximum: MAX_LIMIT,
+      }),
+    ),
+    source: Type.Optional(
+      Type.Union([Type.Literal('web'), Type.Literal('wikipedia')], {
+        description: 'Which engine to query. Defaults to "web" (DuckDuckGo).',
+      }),
+    ),
+  }),
+
+  async execute(_toolCallId, params, signal) {
+    const query = params.query.trim()
+    if (!query) {
+      return errorResult('Query is empty.')
+    }
+    const limit = clampLimit(params.limit)
+    const source = params.source ?? 'web'
+
+    try {
+      const results =
+        source === 'wikipedia' ? await wikipediaSearch(query, limit, signal) : await ddgSearch(query, limit, signal)
+      return successResult(query, source, results)
+    } catch (error) {
+      if (error instanceof DdgCaptchaError) {
+        return errorResult(error.message)
+      }
+      const message = error instanceof Error ? error.message : String(error)
+      return errorResult(`Search failed: ${message}`)
+    }
+  },
+})
+
+function clampLimit(value: number | undefined): number {
+  if (value === undefined) return DEFAULT_LIMIT
+  return Math.min(Math.max(1, Math.floor(value)), MAX_LIMIT)
+}
+
+function successResult(query: string, source: 'web' | 'wikipedia', results: DdgResult[] | WikipediaResult[]) {
+  const details: WebsearchDetails = { query, source, count: results.length, results }
+  if (results.length === 0) {
+    return {
+      content: [{ type: 'text' as const, text: `No results for "${query}" on ${source}.` }],
+      details,
+    }
+  }
+
+  const lines = [`Search results for "${query}" (${source}, ${results.length}):`, '']
+  results.forEach((result, index) => {
+    lines.push(`${index + 1}. ${result.title}`)
+    lines.push(`   ${result.url}`)
+    if (result.snippet) lines.push(`   ${result.snippet}`)
+    lines.push('')
+  })
+
+  return {
+    content: [{ type: 'text' as const, text: lines.join('\n').trimEnd() }],
+    details,
+  }
+}
+
+function errorResult(message: string) {
+  const details: WebsearchDetails = { query: '', source: 'none', count: 0, results: [], error: true, message }
+  return {
+    content: [{ type: 'text' as const, text: message }],
+    details,
+  }
+}

package/src/agent/tools/wikipedia.ts

@@ -0,0 +1,52 @@
+// Wikipedia OpenSearch API: free, official, no key, no rate limit on the free tier.
+// Returns a JSON tuple [query, titles[], descriptions[], urls[]]. Descriptions are
+// usually empty strings, so we don't expose them.
+
+const OPENSEARCH_URL = 'https://en.wikipedia.org/w/api.php'
+
+export type WikipediaResult = {
+  title: string
+  url: string
+  snippet: string
+}
+
+export async function wikipediaSearch(query: string, limit: number, signal?: AbortSignal): Promise<WikipediaResult[]> {
+  const params = new URLSearchParams({
+    action: 'opensearch',
+    search: query,
+    limit: String(limit),
+    format: 'json',
+    namespace: '0',
+  })
+  const response = await fetch(`${OPENSEARCH_URL}?${params.toString()}`, {
+    headers: {
+      'User-Agent': 'TypeClaw/0.1 (https://github.com/devxoul/typeclaw)',
+      Accept: 'application/json',
+    },
+    signal,
+  })
+  if (!response.ok) {
+    throw new Error(`Wikipedia HTTP ${response.status} ${response.statusText}`)
+  }
+  const json = (await response.json()) as unknown
+  return parseOpenSearch(json)
+}
+
+export function parseOpenSearch(json: unknown): WikipediaResult[] {
+  if (!Array.isArray(json) || json.length < 4) return []
+  const titles = asStringArray(json[1])
+  const descriptions = asStringArray(json[2])
+  const urls = asStringArray(json[3])
+  const results: WikipediaResult[] = []
+  for (let i = 0; i < titles.length; i++) {
+    const title = titles[i]
+    const url = urls[i]
+    if (!title || !url) continue
+    results.push({ title, url, snippet: descriptions[i] ?? '' })
+  }
+  return results
+}
+
+function asStringArray(value: unknown): string[] {
+  return Array.isArray(value) ? value.filter((item): item is string => typeof item === 'string') : []
+}