typeclaw 0.48.3 → 0.48.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/package.json +2 -2
  2. package/scripts/dump-system-prompt.ts +146 -11
  3. package/src/agent/index.ts +6 -1
  4. package/src/agent/live-subagents.ts +15 -0
  5. package/src/agent/plugin-tools.ts +7 -0
  6. package/src/agent/session-origin.ts +104 -16
  7. package/src/agent/subagent-completion-reminder.ts +1 -1
  8. package/src/agent/subagent-drain.ts +11 -1
  9. package/src/agent/subagents.ts +52 -7
  10. package/src/agent/tool-file-safety.ts +14 -4
  11. package/src/agent/tools/runtime-notice.ts +29 -0
  12. package/src/agent/tools/spawn-subagent.ts +88 -7
  13. package/src/agent/tools/subagent-cancel.ts +1 -0
  14. package/src/agent/tools/subagent-output.ts +7 -2
  15. package/src/agent/tools/webfetch/adaptive-html.ts +319 -0
  16. package/src/agent/tools/webfetch/html-lexical.ts +841 -0
  17. package/src/agent/tools/webfetch/inline-style-visibility.ts +281 -0
  18. package/src/agent/tools/webfetch/strategies/readability.ts +306 -5
  19. package/src/agent/tools/webfetch/tool.ts +45 -2
  20. package/src/agent/tools/webfetch/types.ts +7 -0
  21. package/src/agent/turn-runner.ts +7 -3
  22. package/src/bundled-plugins/github-cli-auth/gh-command.ts +187 -8
  23. package/src/bundled-plugins/github-cli-auth/git-command.ts +497 -652
  24. package/src/bundled-plugins/memory/load-memory.ts +23 -27
  25. package/src/bundled-plugins/reviewer/reviewer.ts +32 -5
  26. package/src/bundled-plugins/reviewer/skills/code-review.ts +5 -3
  27. package/src/bundled-plugins/reviewer/skills/doc-review.ts +1 -1
  28. package/src/bundled-plugins/reviewer/skills/plan-review.ts +1 -1
  29. package/src/bundled-plugins/security/index.ts +1 -0
  30. package/src/bundled-plugins/security/policies/private-surface-read.ts +74 -18
  31. package/src/channels/router.ts +32 -47
  32. package/src/cli/hostd.ts +1 -1
  33. package/src/hostd/kakao-renewal-manager.ts +23 -11
  34. package/src/plugin/index.ts +1 -0
  35. package/src/plugin/types.ts +9 -10
  36. package/src/run/channel-session-factory.ts +3 -1
  37. package/src/run/index.ts +29 -29
  38. package/src/run/subagent-coalescing.ts +24 -0
  39. package/src/secrets/kakao-renewal.ts +123 -20
  40. package/src/server/index.ts +4 -1
  41. package/src/skills/typeclaw-channel-github/SKILL.md +6 -6
  42. package/src/skills/typeclaw-channel-instagram/SKILL.md +1 -1
  43. package/src/skills/typeclaw-channel-kakaotalk/SKILL.md +1 -1
  44. package/src/skills/typeclaw-channel-line/SKILL.md +1 -1
@@ -1,6 +1,7 @@
1
1
  import { Type } from '@mariozechner/pi-ai'
2
2
  import { defineTool } from '@mariozechner/pi-coding-agent'
3
3
 
4
+ import { extractAdaptiveHtml, validateAdaptiveHtmlComplexity } from './adaptive-html'
4
5
  import { fetchWithLimits, normalizeUrl, parseMimeType, WebFetchError } from './fetch'
5
6
  import { applyGrep, GrepError } from './strategies/grep'
6
7
  import { applyJq, JqError } from './strategies/jq'
@@ -14,6 +15,7 @@ import {
14
15
  MAX_TIMEOUT_SECONDS,
15
16
  OUTPUT_CAPS,
16
17
  type WebFetchDetails,
18
+ type HtmlExtractionAttempt,
17
19
  } from './types'
18
20
 
19
21
  const STRATEGY_VALUES = ['readability', 'jq', 'selector', 'grep', 'snapshot', 'raw'] as const
@@ -32,7 +34,12 @@ export const webFetchTool = defineTool({
32
34
  'fingerprinting (mouse/scroll/timing), interactive CAPTCHAs, or IP-reputation blocks — a 403 from those ' +
33
35
  'layers is expected and unrecoverable from this tool. ' +
34
36
  'Strategy guide:\n' +
35
- '- "readability": extract article content as markdown (blogs, docs, news). Default for HTML.\n' +
37
+ '- "readability": extract article content as markdown (blogs, docs, news). Default for HTML — when auto-detected, ' +
38
+ 'a linear DOM-complexity preflight (5 MiB scan, 20,000 tags, 256 nesting levels) runs before Readability; a ' +
39
+ 'Readability miss (e.g. a JS app shell with no visible content) is rescued by scanning for a bounded schema.org ' +
40
+ 'Article/NewsArticle/BlogPosting JSON-LD block before giving up; if neither yields meaningful text the error says ' +
41
+ 'browser retrieval may be needed. Passing `strategy: "readability"` explicitly skips adaptive JSON-LD/classification ' +
42
+ "and returns Readability's result as-is, even when empty, but still shares parser and deterministic visibility safety checks.\n" +
36
43
  '- "jq": query JSON APIs (npm registry, GitHub API). Pass `query` (e.g. ".items[].name").\n' +
37
44
  '- "selector": extract text from elements matching a CSS selector. Pass `selector` (e.g. ".price").\n' +
38
45
  '- "grep": filter lines by regex with optional `before`/`after` context. Pass `pattern`.\n' +
@@ -125,8 +132,41 @@ export const webFetchTool = defineTool({
125
132
  }
126
133
 
127
134
  let output: string
135
+ let extractionAttempts: HtmlExtractionAttempt[] | undefined
128
136
  try {
129
- output = await runStrategy(strategy, response.body, response.finalUrl, params)
137
+ if (strategy === 'readability' && autoDetected) {
138
+ const extracted = await extractAdaptiveHtml(response.body, response.finalUrl)
139
+ extractionAttempts = extracted.attempts
140
+ if (extracted.kind === 'error') {
141
+ return errorResult(normalizedUrl, extracted.message, {
142
+ startedAt,
143
+ finalUrl: response.finalUrl,
144
+ contentType: response.contentType,
145
+ httpStatus: response.httpStatus,
146
+ bytesIn: response.bytesIn,
147
+ strategy,
148
+ autoDetected,
149
+ extractionAttempts,
150
+ })
151
+ }
152
+ output = extracted.output
153
+ } else {
154
+ if (strategy === 'readability') {
155
+ const complexity = validateAdaptiveHtmlComplexity(response.body)
156
+ if (!complexity.valid) {
157
+ return errorResult(normalizedUrl, complexity.reason, {
158
+ startedAt,
159
+ finalUrl: response.finalUrl,
160
+ contentType: response.contentType,
161
+ httpStatus: response.httpStatus,
162
+ bytesIn: response.bytesIn,
163
+ strategy,
164
+ autoDetected,
165
+ })
166
+ }
167
+ }
168
+ output = await runStrategy(strategy, response.body, response.finalUrl, params)
169
+ }
130
170
  } catch (error) {
131
171
  const message = error instanceof Error ? error.message : String(error)
132
172
  return errorResult(normalizedUrl, message, {
@@ -152,6 +192,7 @@ export const webFetchTool = defineTool({
152
192
  bytesOut: byteLength(capped.text),
153
193
  truncated: capped.truncated,
154
194
  durationMs: Date.now() - startedAt,
195
+ ...(extractionAttempts === undefined ? {} : { extractionAttempts }),
155
196
  ...(response.antibotWarmup?.triggered
156
197
  ? {
157
198
  antibotWarmup: {
@@ -286,6 +327,8 @@ function errorResult(
286
327
  bytesOut: byteLength(message),
287
328
  truncated: false,
288
329
  durationMs: Date.now() - startedAt,
330
+ ...(rest.antibotWarmup === undefined ? {} : { antibotWarmup: rest.antibotWarmup }),
331
+ ...(rest.extractionAttempts === undefined ? {} : { extractionAttempts: rest.extractionAttempts }),
289
332
  error: true,
290
333
  message,
291
334
  }
@@ -7,6 +7,12 @@ export type AntibotWarmupDetails = {
7
7
  replayStatus?: number
8
8
  }
9
9
 
10
+ export type HtmlExtractionAttempt = {
11
+ route: 'readability' | 'json_ld'
12
+ outcome: 'success' | 'unusable' | 'error'
13
+ reason: string
14
+ }
15
+
10
16
  export type WebFetchDetails = {
11
17
  url: string
12
18
  finalUrl: string
@@ -19,6 +25,7 @@ export type WebFetchDetails = {
19
25
  truncated: boolean
20
26
  durationMs: number
21
27
  antibotWarmup?: AntibotWarmupDetails
28
+ extractionAttempts?: HtmlExtractionAttempt[]
22
29
  error?: boolean
23
30
  message?: string
24
31
  }
@@ -37,9 +37,13 @@ type AttemptActivity = {
37
37
  startedToolExecution: boolean
38
38
  }
39
39
 
40
- // SDK-level attempts already contain pi-ai's internal fetch recovery. Bounding
41
- // only repeated no-header outcomes limits dead air without timing out healthy
42
- // reasoning or tool execution.
40
+ // SDK-level attempts already contain pi-ai's internal fetch recovery. Three is
41
+ // the safe minimum for a fallback chain: primary attempt + its one ordinary
42
+ // transient replay + one cross-ref attempt. Lowering it would weaken either the
43
+ // retained transient retry or fallback; raising it would recreate the repeated
44
+ // TTFB bursts reviewer coalescing is intended to prevent from multiplying.
45
+ // Patient overload retries do not apply to observer TTFB errors, so this envelope
46
+ // already bounds deep-review no-progress without a reviewer-specific retry fork.
43
47
  const MAX_NO_PROGRESS_ATTEMPTS = 3
44
48
  const MAX_CUMULATIVE_NO_PROGRESS_MS = 60_000
45
49
 
@@ -79,7 +79,7 @@ const COMPOSITION_REASON =
79
79
  'newlines, redirections, command/process substitution, subshells, heredocs, or unquoted ' +
80
80
  '`$` expansion (any sibling process or expansion would inherit the token and could ' +
81
81
  'exfiltrate it). One exception is allowed: a trailing reader pipeline `gh … | <reader>` ' +
82
- 'where every downstream stage is a stdin-only reader (`jq`, `cat`, `wc`, `sort`, `uniq`) ' +
82
+ 'where every downstream stage is a stdin-only reader (`jq`, `grep`, `cat`, `wc`, `sort`, `uniq`) ' +
83
83
  'with no file operand — e.g. `gh api repos/o/r | jq .`. jq/JSON metacharacters are also ' +
84
84
  "fine INSIDE single quotes, e.g. `gh api repos/o/r --jq '.[] | {id}'`. File-backed " +
85
85
  '`--input`, `--body-file`, templates, aliases, extensions, and auth/config commands remain unavailable.'
@@ -262,7 +262,7 @@ function isAuthStatusTokenDisplayFlag(arg: string): boolean {
262
262
 
263
263
  export function canInjectPatIntoPassThroughGh(command: string): boolean {
264
264
  if (containsReviewGraphqlMutation(command)) return false
265
- if (command.includes('\\')) return false
265
+ if (hasUnsafeGhStageBackslash(command)) return false
266
266
  if (!isSingleBareGhCommand(command)) return false
267
267
  const tokens = tokenize(command)
268
268
  const starts = findGhInvocations(tokens)
@@ -410,7 +410,7 @@ function isCredentialSafeGhCommand(command: string): boolean {
410
410
  const stages = splitTopLevelPipeStages(command)
411
411
  if (stages === null || stages.length === 0) return false
412
412
  const ghStage = (stages[0] as string).trim()
413
- if (!isSingleBareGhCommand(ghStage) || ghStage.includes('\\')) return false
413
+ if (!isSingleBareGhCommand(ghStage) || hasUnsafeGhStageBackslash(ghStage)) return false
414
414
  const tokens = tokenize(ghStage)
415
415
  return tokens[0] === 'gh' && isCredentialSafeGhArgs(tokens.slice(1))
416
416
  }
@@ -500,9 +500,11 @@ function isSafeGhJqFilter(filter: string): boolean {
500
500
  // has gh's output) — they cannot open their own network/file/process sink, so a
501
501
  // `gh <repo> | <reader>` pipeline cannot exfiltrate the minted token to a third
502
502
  // party. EXCLUDED on purpose: awk (system()/getline|cmd/inet), sed (GNU `e`
503
- // shell-exec), tee/xargs (write/spawn), less (`!cmd`), and grep/head/tail (their
503
+ // shell-exec), tee/xargs (write/spawn), less (`!cmd`), and head/tail (their
504
504
  // file-operand forms are too easy to abuse and not worth the parser risk yet).
505
- const READER_ALLOWLIST = new Set(['jq', 'cat', 'wc', 'sort', 'uniq'])
505
+ // grep is admitted through a dedicated grammar that requires a pattern and
506
+ // proves there is no file operand or pattern-file/recursive option.
507
+ const READER_ALLOWLIST = new Set(['jq', 'grep', 'cat', 'wc', 'sort', 'uniq'])
506
508
 
507
509
  // STRICT per-command flag allowlists. We allow ONLY flags known to be pure
508
510
  // stdin-shaping (no file/program operand). This is allow-known-good, not
@@ -561,6 +563,22 @@ const JQ_SAFE_VALUE_LONG: Record<string, number> = {
561
563
  // (module path) are the fatal ones — and any unknown char also rejects.
562
564
  const JQ_SAFE_BOOLEAN_SHORT = new Set(['r', 'c', 's', 'n', 'e', 'a', 'S', 'R', 'j', 'C', 'M', 'b'])
563
565
 
566
+ const GREP_SAFE_BOOLEAN_LONG = new Set([
567
+ '--basic-regexp',
568
+ '--extended-regexp',
569
+ '--fixed-strings',
570
+ '--ignore-case',
571
+ '--invert-match',
572
+ '--line-number',
573
+ '--count',
574
+ '--only-matching',
575
+ '--quiet',
576
+ '--silent',
577
+ '--word-regexp',
578
+ '--line-regexp',
579
+ ])
580
+ const GREP_SAFE_BOOLEAN_SHORT = new Set(['E', 'F', 'G', 'i', 'v', 'n', 'c', 'o', 'q', 'w', 'x'])
581
+
564
582
  // A reader stage is safe only if it is an allowlisted command using ONLY its
565
583
  // known stdin-shaping flags, with no file operand. Backslashes are rejected
566
584
  // outright: our tokenizer does not model shell backslash escaping, so a
@@ -568,12 +586,13 @@ const JQ_SAFE_BOOLEAN_SHORT = new Set(['r', 'c', 's', 'n', 'e', 'a', 'S', 'R', '
568
586
  // as the forbidden flag — an allowlist-bypass. Rejecting `\` closes that gap.
569
587
  function isStdinOnlyReaderStage(stage: string): boolean {
570
588
  if (containsShellActiveMetachar(stage)) return false
571
- if (stage.includes('\\')) return false
572
589
  const tokens = splitStageTokens(stage)
573
590
  const cmd = tokens[0]
574
591
  if (cmd === undefined || !READER_ALLOWLIST.has(cmd)) return false
575
592
 
576
- if (cmd === 'jq') return isStdinOnlyJqStage(tokens)
593
+ if (cmd === 'jq') return !hasUnsafeJqStageBackslash(stage) && isStdinOnlyJqStage(tokens)
594
+ if (stage.includes('\\')) return false
595
+ if (cmd === 'grep') return isStdinOnlyGrepStage(tokens)
577
596
 
578
597
  const allowedFlags = READER_BOOLEAN_FLAGS[cmd]
579
598
  if (allowedFlags === undefined) return false
@@ -585,6 +604,65 @@ function isStdinOnlyReaderStage(stage: string): boolean {
585
604
  return true
586
605
  }
587
606
 
607
+ // grep reads stdin only when its argv contains one positional pattern OR one or
608
+ // more -e/--regexp patterns, followed by no positional file operands. Keep the
609
+ // option surface intentionally small: unknown options fail closed, especially
610
+ // -f/--file, recursive traversal, and include/exclude selectors.
611
+ function isStdinOnlyGrepStage(tokens: readonly string[]): boolean {
612
+ let patternCount = 0
613
+ let usesRegexpFlag = false
614
+ let sawPositionalPattern = false
615
+ for (let i = 1; i < tokens.length; i++) {
616
+ const tok = tokens[i] as string
617
+ if (tok === '--') return false
618
+ if (sawPositionalPattern && tok.startsWith('-')) return false
619
+ if (tok === '-e' || tok === '--regexp') {
620
+ if (sawPositionalPattern) return false
621
+ const pattern = tokens[i + 1]
622
+ if (pattern === undefined) return false
623
+ patternCount += 1
624
+ usesRegexpFlag = true
625
+ i += 1
626
+ continue
627
+ }
628
+ if (tok.startsWith('--regexp=')) {
629
+ if (sawPositionalPattern) return false
630
+ patternCount += 1
631
+ usesRegexpFlag = true
632
+ continue
633
+ }
634
+ if (tok.startsWith('--')) {
635
+ if (!GREP_SAFE_BOOLEAN_LONG.has(tok)) return false
636
+ continue
637
+ }
638
+ if (tok.startsWith('-') && tok.length > 1) {
639
+ const shortOptions = tok.slice(1)
640
+ for (let shortIndex = 0; shortIndex < shortOptions.length; shortIndex++) {
641
+ const ch = shortOptions[shortIndex] as string
642
+ if (ch === 'e') {
643
+ if (sawPositionalPattern) return false
644
+ patternCount += 1
645
+ usesRegexpFlag = true
646
+ // grep treats `e` as a value-taking terminal option: the rest of the
647
+ // cluster is its pattern (`-neerror`), or the next token is (`-ne x`).
648
+ // No later cluster character can therefore become another option.
649
+ if (shortIndex + 1 < shortOptions.length) break
650
+ const pattern = tokens[i + 1]
651
+ if (pattern === undefined) return false
652
+ i += 1
653
+ break
654
+ }
655
+ if (!GREP_SAFE_BOOLEAN_SHORT.has(ch)) return false
656
+ }
657
+ continue
658
+ }
659
+ if (usesRegexpFlag || patternCount > 0) return false
660
+ patternCount = 1
661
+ sawPositionalPattern = true
662
+ }
663
+ return patternCount > 0
664
+ }
665
+
588
666
  // jq must run pure-stdin: only known stdin-shaping flags, and EXACTLY one
589
667
  // positional (the filter). A second positional is an input FILE jq would open
590
668
  // (`jq . /proc/self/environ` reads that file), so it is rejected. The filter is
@@ -641,10 +719,18 @@ function analyzeReaderPipeline(command: string, stripRepoFlag: boolean): string
641
719
  }
642
720
 
643
721
  const rewrittenGh = stripRepoFlag ? stripRepoFlagFromCommand(ghStage) : ghStage
644
- const rewrittenReaders = stages.slice(1).map((s) => `/usr/bin/env -u GH_TOKEN -u GITHUB_TOKEN ${s.trim()}`)
722
+ const rewrittenReaders = stages.slice(1).map(rewriteReaderStage)
645
723
  return [rewrittenGh, ...rewrittenReaders].join(' | ')
646
724
  }
647
725
 
726
+ function rewriteReaderStage(stage: string): string {
727
+ const trimmed = stage.trim()
728
+ if (splitStageTokens(trimmed)[0] === 'grep') {
729
+ return `/usr/bin/env -u GH_TOKEN -u GITHUB_TOKEN -u GREP_OPTIONS /usr/bin/grep${trimmed.slice('grep'.length)}`
730
+ }
731
+ return `/usr/bin/env -u GH_TOKEN -u GITHUB_TOKEN ${trimmed}`
732
+ }
733
+
648
734
  // Quote-aware split on top-level `|`. Returns null if any OTHER shell-active
649
735
  // metachar appears outside single quotes (`;` `&` `<` `>` backtick `$` `(` `)`
650
736
  // `{` `}` newline) or if a `||`/`|&` is seen — those are not simple pipelines.
@@ -742,6 +828,99 @@ function splitStageTokens(stage: string): string[] {
742
828
  return tokens
743
829
  }
744
830
 
831
+ // Backslashes are shell-active outside single quotes, and the argv-ish tokenizers
832
+ // below deliberately do not interpret shell escapes. The one safe use we support
833
+ // is jq syntax such as `split("\\n")` when the complete filter operand is
834
+ // single-quoted: the shell passes that backslash literally to jq, so it cannot
835
+ // disguise a forbidden gh/jq flag or file operand.
836
+ function hasUnsafeGhStageBackslash(stage: string): boolean {
837
+ if (!stage.includes('\\')) return false
838
+ const rawTokens = splitRawStageTokens(stage)
839
+ if (rawTokens === null) return true
840
+
841
+ for (let i = 0; i < rawTokens.length; i++) {
842
+ const raw = rawTokens[i] as string
843
+ if (!raw.includes('\\')) continue
844
+ const previous = rawTokens[i - 1]
845
+ if ((previous === '--jq' || previous === '-q') && isFullySingleQuoted(raw)) continue
846
+ const attachedFilter = raw.startsWith('--jq=')
847
+ ? raw.slice('--jq='.length)
848
+ : raw.startsWith('-q=')
849
+ ? raw.slice(3)
850
+ : null
851
+ if (attachedFilter !== null && isFullySingleQuoted(attachedFilter)) continue
852
+ return true
853
+ }
854
+ return false
855
+ }
856
+
857
+ function hasUnsafeJqStageBackslash(stage: string): boolean {
858
+ if (!stage.includes('\\')) return false
859
+ const rawTokens = splitRawStageTokens(stage)
860
+ if (rawTokens === null || rawTokens[0] !== 'jq') return true
861
+
862
+ let filterIndex: number | null = null
863
+ for (let i = 1; i < rawTokens.length; i++) {
864
+ const token = stripPairedQuotes(rawTokens[i] as string)
865
+ if (token === '--') return true
866
+ if (token.startsWith('--')) {
867
+ const consume = JQ_SAFE_VALUE_LONG[token]
868
+ if (consume !== undefined) i += consume
869
+ continue
870
+ }
871
+ if (token.startsWith('-') && token.length > 1) continue
872
+ filterIndex = i
873
+ break
874
+ }
875
+
876
+ for (let i = 1; i < rawTokens.length; i++) {
877
+ const raw = rawTokens[i] as string
878
+ if (!raw.includes('\\')) continue
879
+ if (i === filterIndex && isFullySingleQuoted(raw)) continue
880
+ return true
881
+ }
882
+ return false
883
+ }
884
+
885
+ function splitRawStageTokens(stage: string): string[] | null {
886
+ const tokens: string[] = []
887
+ let current = ''
888
+ let quote: '"' | "'" | null = null
889
+ for (const ch of stage) {
890
+ if (quote !== null) {
891
+ current += ch
892
+ if (ch === quote) quote = null
893
+ continue
894
+ }
895
+ if (ch === '"' || ch === "'") {
896
+ quote = ch
897
+ current += ch
898
+ continue
899
+ }
900
+ if (ch === ' ' || ch === '\t') {
901
+ if (current !== '') {
902
+ tokens.push(current)
903
+ current = ''
904
+ }
905
+ continue
906
+ }
907
+ current += ch
908
+ }
909
+ if (quote !== null) return null
910
+ if (current !== '') tokens.push(current)
911
+ return tokens
912
+ }
913
+
914
+ function isFullySingleQuoted(raw: string): boolean {
915
+ return raw.length >= 2 && raw.startsWith("'") && raw.endsWith("'") && !raw.slice(1, -1).includes("'")
916
+ }
917
+
918
+ function stripPairedQuotes(raw: string): string {
919
+ if (raw.length < 2) return raw
920
+ const first = raw[0]
921
+ return first === raw[raw.length - 1] && (first === '"' || first === "'") ? raw.slice(1, -1) : raw
922
+ }
923
+
745
924
  // Removes an unquoted `-R`/`--repo` flag (and its repo-slug value) from a single
746
925
  // bare command, preserving everything else byte-for-byte. Quote-aware so a `-R`
747
926
  // inside a quoted `-f query='...'` value is never touched; a repo slug is