typeclaw 0.48.4 → 0.48.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/package.json +1 -1
  2. package/src/agent/index.ts +6 -1
  3. package/src/agent/live-subagents.ts +15 -0
  4. package/src/agent/session-origin.ts +104 -16
  5. package/src/agent/subagent-completion-reminder.ts +1 -1
  6. package/src/agent/subagent-drain.ts +11 -1
  7. package/src/agent/subagents.ts +52 -7
  8. package/src/agent/tools/spawn-subagent.ts +88 -7
  9. package/src/agent/tools/subagent-cancel.ts +1 -0
  10. package/src/agent/tools/subagent-output.ts +7 -2
  11. package/src/agent/tools/webfetch/adaptive-html.ts +319 -0
  12. package/src/agent/tools/webfetch/html-lexical.ts +841 -0
  13. package/src/agent/tools/webfetch/inline-style-visibility.ts +281 -0
  14. package/src/agent/tools/webfetch/strategies/readability.ts +306 -5
  15. package/src/agent/tools/webfetch/tool.ts +45 -2
  16. package/src/agent/tools/webfetch/types.ts +7 -0
  17. package/src/agent/turn-runner.ts +7 -3
  18. package/src/bundled-plugins/github-cli-auth/gh-command.ts +187 -8
  19. package/src/bundled-plugins/reviewer/reviewer.ts +32 -5
  20. package/src/bundled-plugins/reviewer/skills/code-review.ts +5 -3
  21. package/src/bundled-plugins/reviewer/skills/doc-review.ts +1 -1
  22. package/src/bundled-plugins/reviewer/skills/plan-review.ts +1 -1
  23. package/src/plugin/types.ts +0 -10
  24. package/src/run/channel-session-factory.ts +3 -1
  25. package/src/run/index.ts +29 -29
  26. package/src/run/subagent-coalescing.ts +24 -0
  27. package/src/server/index.ts +4 -1
  28. package/src/skills/typeclaw-channel-github/SKILL.md +6 -6
  29. package/src/skills/typeclaw-channel-instagram/SKILL.md +1 -1
  30. package/src/skills/typeclaw-channel-kakaotalk/SKILL.md +1 -1
  31. package/src/skills/typeclaw-channel-line/SKILL.md +1 -1
@@ -0,0 +1,319 @@
1
+ import { walkHtmlLexically } from './html-lexical'
2
+ import { applyReadabilityDetailed, type ReadabilityDetailedResult } from './strategies/readability'
3
+ import type { HtmlExtractionAttempt } from './types'
4
+
5
+ export type AdaptiveHtmlResult =
6
+ | { kind: 'success'; output: string; attempts: HtmlExtractionAttempt[] }
7
+ | { kind: 'error'; message: string; attempts: HtmlExtractionAttempt[] }
8
+
9
+ type AdaptiveHtmlDependencies = {
10
+ applyReadabilityDetailed: typeof applyReadabilityDetailed
11
+ }
12
+
13
+ export const MAX_ADAPTIVE_HTML_SCAN_BYTES = 5 * 1024 * 1024
14
+ export const MAX_ADAPTIVE_HTML_TAG_COUNT = 20_000
15
+ export const MAX_ADAPTIVE_HTML_NESTING_DEPTH = 256
16
+ export const MAX_ADAPTIVE_HTML_ATTRIBUTES_PER_TAG = 1_024
17
+ export const MAX_ADAPTIVE_HTML_START_TAG_CHARS = 64 * 1024
18
+
19
+ const MAX_JSON_LD_SCAN_BYTES = 1_000_000
20
+ const MAX_JSON_LD_BLOCKS = 32
21
+ const MAX_JSON_LD_BLOCK_BYTES = 256_000
22
+ const MAX_JSON_LD_VALUE_CHARS = 200_000
23
+ const ARTICLE_TYPES = new Set(['Article', 'NewsArticle', 'BlogPosting'])
24
+ const EMPTY_READABILITY_OUTPUT = 'Readability extracted no content from this page.'
25
+ const COMPLEXITY_LIMIT_REASON = 'adaptive HTML complexity limit exceeded'
26
+ const MALFORMED_HTML_REASON = 'adaptive HTML structure could not be validated'
27
+ // A bounded character floor accepts substantial script-free fallback while keeping short chrome fail-closed.
28
+ const SUBSTANTIAL_VISIBLE_TEXT_CHARACTERS = 512
29
+ const ENCODER = new TextEncoder()
30
+
31
+ export async function extractAdaptiveHtml(
32
+ html: string,
33
+ url: string,
34
+ dependencies: AdaptiveHtmlDependencies = { applyReadabilityDetailed },
35
+ ): Promise<AdaptiveHtmlResult> {
36
+ const attempts: HtmlExtractionAttempt[] = []
37
+
38
+ const complexity = validateAdaptiveHtmlComplexity(html)
39
+ if (!complexity.valid) {
40
+ attempts.push({ route: 'readability', outcome: 'error', reason: complexity.reason })
41
+ } else {
42
+ try {
43
+ const readability = await dependencies.applyReadabilityDetailed(html, url)
44
+ const verdict = classifyReadability(readability)
45
+ if (verdict.useful) {
46
+ attempts.push({ route: 'readability', outcome: 'success', reason: verdict.reason })
47
+ return { kind: 'success', output: readability.output, attempts }
48
+ }
49
+ attempts.push({ route: 'readability', outcome: 'unusable', reason: verdict.reason })
50
+ } catch {
51
+ attempts.push({ route: 'readability', outcome: 'error', reason: 'Readability extraction failed' })
52
+ }
53
+ }
54
+
55
+ try {
56
+ const output = extractJsonLdArticle(html)
57
+ if (output !== null) {
58
+ attempts.push({ route: 'json_ld', outcome: 'success', reason: 'extracted schema.org article content' })
59
+ return { kind: 'success', output, attempts }
60
+ }
61
+ attempts.push({ route: 'json_ld', outcome: 'unusable', reason: 'no schema.org article content found' })
62
+ } catch {
63
+ attempts.push({ route: 'json_ld', outcome: 'error', reason: 'schema.org extraction failed' })
64
+ }
65
+
66
+ const extractorFailed = attempts.some(({ outcome }) => outcome === 'error')
67
+ return {
68
+ kind: 'error',
69
+ message: extractorFailed
70
+ ? 'An HTML extractor failed, and no public article content was recovered. Browser retrieval may be needed.'
71
+ : 'Fetched HTML did not contain public article content. The page may require client-side rendering or access controls; browser retrieval may be needed.',
72
+ attempts,
73
+ }
74
+ }
75
+
76
+ export type AdaptiveHtmlComplexityResult = { valid: true } | { valid: false; reason: string }
77
+
78
+ export function validateAdaptiveHtmlComplexity(html: string): AdaptiveHtmlComplexityResult {
79
+ if (byteLength(html) > MAX_ADAPTIVE_HTML_SCAN_BYTES) return complexityFailure()
80
+ const result = walkHtmlLexically(html, {
81
+ maxTags: MAX_ADAPTIVE_HTML_TAG_COUNT,
82
+ maxDepth: MAX_ADAPTIVE_HTML_NESTING_DEPTH,
83
+ maxAttributesPerTag: MAX_ADAPTIVE_HTML_ATTRIBUTES_PER_TAG,
84
+ maxStartTagChars: MAX_ADAPTIVE_HTML_START_TAG_CHARS,
85
+ })
86
+ if (result.valid) return result
87
+ return result.failure === 'limit' ? complexityFailure() : { valid: false, reason: MALFORMED_HTML_REASON }
88
+ }
89
+
90
+ export function scanJsonLdScripts(html: string): string[] {
91
+ if (byteLength(html) > MAX_ADAPTIVE_HTML_SCAN_BYTES) return []
92
+ const payloadCharacterLimit = sliceByBytes(html, MAX_JSON_LD_SCAN_BYTES).length
93
+ const blocks: string[] = []
94
+ let embeddedStylesheetSeen = false
95
+ let inspected = 0
96
+ const result = walkHtmlLexically(html, {
97
+ maxTags: MAX_ADAPTIVE_HTML_TAG_COUNT,
98
+ maxDepth: MAX_ADAPTIVE_HTML_NESTING_DEPTH,
99
+ maxAttributesPerTag: MAX_ADAPTIVE_HTML_ATTRIBUTES_PER_TAG,
100
+ maxStartTagChars: MAX_ADAPTIVE_HTML_START_TAG_CHARS,
101
+ onStartTag: ({ name }) => {
102
+ if (name === 'style') embeddedStylesheetSeen = true
103
+ },
104
+ onRawElement: (element) => {
105
+ if (element.name === 'style') return
106
+ if (
107
+ element.name !== 'script' ||
108
+ element.metadataSuppressed ||
109
+ element.typeAttribute !== 'application/ld+json' ||
110
+ element.bodyEnd > payloadCharacterLimit ||
111
+ inspected >= MAX_JSON_LD_BLOCKS
112
+ ) {
113
+ return
114
+ }
115
+ inspected++
116
+ const source = html.slice(element.bodyStart, element.bodyEnd).trim()
117
+ if (source && byteLength(source) <= MAX_JSON_LD_BLOCK_BYTES) blocks.push(source)
118
+ },
119
+ })
120
+ return result.valid && !embeddedStylesheetSeen ? blocks : []
121
+ }
122
+
123
+ function classifyReadability(result: ReadabilityDetailedResult): { useful: boolean; reason: string } {
124
+ if (result.output === EMPTY_READABILITY_OUTPUT)
125
+ return { useful: false, reason: 'Readability found no visible content' }
126
+
127
+ const hasClearContent =
128
+ result.parsedArticle &&
129
+ (result.hasSemanticContentContainer || result.hasArticleContainer || result.proseElementCount > 0)
130
+ if (hasClearContent) {
131
+ return { useful: true, reason: 'Readability identified article content' }
132
+ }
133
+
134
+ if (result.visibleTextCharacterCount >= SUBSTANTIAL_VISIBLE_TEXT_CHARACTERS) {
135
+ return { useful: true, reason: 'generic body fallback contained substantial visible content' }
136
+ }
137
+
138
+ const scriptRenderedShell =
139
+ !result.hasSemanticContentContainer &&
140
+ result.scriptCount >= 2 &&
141
+ (result.structuredContentElementCount > 0 || result.scriptCount >= 3)
142
+
143
+ if (scriptRenderedShell) {
144
+ return { useful: false, reason: 'generic body fallback looked like a script-rendered shell' }
145
+ }
146
+
147
+ if (result.parsedArticle && result.structuredContentElementCount > 0) {
148
+ return { useful: true, reason: 'Readability identified article content' }
149
+ }
150
+
151
+ if (result.parsedArticle) {
152
+ return { useful: false, reason: 'Readability identified only unstructured page chrome' }
153
+ }
154
+
155
+ return { useful: false, reason: 'generic body fallback contained only short visible content' }
156
+ }
157
+
158
+ function complexityFailure(): AdaptiveHtmlComplexityResult {
159
+ return { valid: false, reason: COMPLEXITY_LIMIT_REASON }
160
+ }
161
+
162
+ function extractJsonLdArticle(html: string): string | null {
163
+ for (const source of scanJsonLdScripts(html)) {
164
+ let parsed: unknown
165
+ try {
166
+ parsed = JSON.parse(source)
167
+ } catch {
168
+ continue
169
+ }
170
+ for (const candidate of articleCandidates(parsed)) {
171
+ const headline = plainText(candidate.headline)
172
+ const body = plainText(candidate.articleBody) || plainText(candidate.description)
173
+ if (!body) continue
174
+ return headline ? `# ${escapeMarkdownHeading(headline)}\n\n${body}` : body
175
+ }
176
+ }
177
+ return null
178
+ }
179
+
180
+ function articleCandidates(value: unknown, depth = 0, inheritedContext?: unknown): Array<Record<string, unknown>> {
181
+ if (depth > 4) return []
182
+ if (Array.isArray(value)) return value.flatMap((item) => articleCandidates(item, depth + 1, inheritedContext))
183
+ if (!isRecord(value)) return []
184
+
185
+ const context = Object.hasOwn(value, '@context') ? value['@context'] : inheritedContext
186
+ const candidates = isArticleType(value['@type'], context) ? [value] : []
187
+ const graph = value['@graph']
188
+ if (graph !== undefined) candidates.push(...articleCandidates(graph, depth + 1, context))
189
+ return candidates
190
+ }
191
+
192
+ function isArticleType(value: unknown, context: unknown): boolean {
193
+ const values = Array.isArray(value) ? value : [value]
194
+ return values.some((item) => {
195
+ if (typeof item !== 'string') return false
196
+ const absoluteSchemaType = schemaOrgName(item)
197
+ if (absoluteSchemaType !== null) return ARTICLE_TYPES.has(absoluteSchemaType)
198
+ if (context === undefined) return ARTICLE_TYPES.has(item)
199
+ const resolved = resolveContextType(item, context)
200
+ return resolved !== null && ARTICLE_TYPES.has(resolved)
201
+ })
202
+ }
203
+
204
+ function schemaOrgName(value: string): string | null {
205
+ const match = /^https?:\/\/schema\.org\/(?:[^/#]+[/#])?([^/#]+)$/iu.exec(value)
206
+ return match?.[1] ?? null
207
+ }
208
+
209
+ function resolveContextType(type: string, context: unknown, depth = 0): string | null {
210
+ return resolveContextDefinition(type, context, depth).value
211
+ }
212
+
213
+ function resolveContextDefinition(
214
+ type: string,
215
+ context: unknown,
216
+ depth: number,
217
+ ): { defined: boolean; value: string | null } {
218
+ if (depth > 4) return { defined: false, value: null }
219
+ if (context === null) return { defined: true, value: null }
220
+ if (typeof context === 'string') {
221
+ return { defined: true, value: isSchemaOrgBase(context) && ARTICLE_TYPES.has(type) ? type : null }
222
+ }
223
+ if (Array.isArray(context)) {
224
+ for (let index = context.length - 1; index >= 0; index--) {
225
+ const resolution = resolveContextDefinition(type, context[index], depth + 1)
226
+ if (resolution.defined) return resolution
227
+ }
228
+ return { defined: false, value: null }
229
+ }
230
+ if (!isRecord(context)) return { defined: false, value: null }
231
+
232
+ const termDefined = Object.hasOwn(context, type)
233
+ const term = context[type]
234
+ const termId = typeof term === 'string' ? term : isRecord(term) ? term['@id'] : undefined
235
+ if (termDefined) {
236
+ const absolute = typeof termId === 'string' ? schemaOrgName(termId) : null
237
+ return { defined: true, value: absolute }
238
+ }
239
+
240
+ const separator = type.indexOf(':')
241
+ if (separator > 0) {
242
+ const prefixName = type.slice(0, separator)
243
+ if (!Object.hasOwn(context, prefixName)) return { defined: false, value: null }
244
+ const prefix = context[prefixName]
245
+ const prefixId = typeof prefix === 'string' ? prefix : isRecord(prefix) ? prefix['@id'] : undefined
246
+ return {
247
+ defined: true,
248
+ value: typeof prefixId === 'string' && isSchemaOrgBase(prefixId) ? type.slice(separator + 1) : null,
249
+ }
250
+ }
251
+
252
+ if (!Object.hasOwn(context, '@vocab')) return { defined: false, value: null }
253
+ const vocabulary = context['@vocab']
254
+ return {
255
+ defined: true,
256
+ value: typeof vocabulary === 'string' && isSchemaOrgBase(vocabulary) ? type : null,
257
+ }
258
+ }
259
+
260
+ function isSchemaOrgBase(value: string): boolean {
261
+ return /^https?:\/\/schema\.org\/?$/iu.test(value)
262
+ }
263
+
264
+ function plainText(value: unknown): string {
265
+ if (typeof value !== 'string') return ''
266
+ return stripTags(value.slice(0, MAX_JSON_LD_VALUE_CHARS))
267
+ .replace(/&(?:nbsp|#160);/giu, ' ')
268
+ .replace(/&amp;/giu, '&')
269
+ .replace(/&lt;/giu, '<')
270
+ .replace(/&gt;/giu, '>')
271
+ .replace(/&quot;/giu, '"')
272
+ .replace(/&#39;|&apos;/giu, "'")
273
+ .replace(/\s+/gu, ' ')
274
+ .trim()
275
+ }
276
+
277
+ function stripTags(value: string): string {
278
+ const output: string[] = []
279
+ let copiedThrough = 0
280
+ let cursor = 0
281
+ while (cursor < value.length) {
282
+ if (value[cursor] !== '<') {
283
+ cursor++
284
+ continue
285
+ }
286
+ output.push(value.slice(copiedThrough, cursor))
287
+ const candidateStart = cursor
288
+ cursor++
289
+ while (cursor < value.length && value[cursor] !== '>') cursor++
290
+ if (cursor >= value.length) {
291
+ output.push(value.slice(candidateStart))
292
+ copiedThrough = value.length
293
+ break
294
+ }
295
+ output.push(' ')
296
+ cursor++
297
+ copiedThrough = cursor
298
+ }
299
+ if (copiedThrough < value.length) output.push(value.slice(copiedThrough))
300
+ return output.join('')
301
+ }
302
+
303
+ function escapeMarkdownHeading(value: string): string {
304
+ return value.replace(/([\\`*_[\]{}()#+.!|>-])/gu, '\\$1')
305
+ }
306
+
307
+ function isRecord(value: unknown): value is Record<string, unknown> {
308
+ return typeof value === 'object' && value !== null
309
+ }
310
+
311
+ function byteLength(value: string): number {
312
+ return ENCODER.encode(value).byteLength
313
+ }
314
+
315
+ function sliceByBytes(value: string, maxBytes: number): string {
316
+ const bytes = ENCODER.encode(value)
317
+ if (bytes.byteLength <= maxBytes) return value
318
+ return new TextDecoder('utf-8', { fatal: false }).decode(bytes.slice(0, maxBytes))
319
+ }