typeclaw 0.48.4 → 0.48.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/agent/index.ts +6 -1
- package/src/agent/live-subagents.ts +15 -0
- package/src/agent/session-origin.ts +104 -16
- package/src/agent/subagent-completion-reminder.ts +1 -1
- package/src/agent/subagent-drain.ts +11 -1
- package/src/agent/subagents.ts +52 -7
- package/src/agent/tools/spawn-subagent.ts +88 -7
- package/src/agent/tools/subagent-cancel.ts +1 -0
- package/src/agent/tools/subagent-output.ts +7 -2
- package/src/agent/tools/webfetch/adaptive-html.ts +319 -0
- package/src/agent/tools/webfetch/html-lexical.ts +841 -0
- package/src/agent/tools/webfetch/inline-style-visibility.ts +281 -0
- package/src/agent/tools/webfetch/strategies/readability.ts +306 -5
- package/src/agent/tools/webfetch/tool.ts +45 -2
- package/src/agent/tools/webfetch/types.ts +7 -0
- package/src/agent/turn-runner.ts +7 -3
- package/src/bundled-plugins/github-cli-auth/gh-command.ts +187 -8
- package/src/bundled-plugins/reviewer/reviewer.ts +32 -5
- package/src/bundled-plugins/reviewer/skills/code-review.ts +5 -3
- package/src/bundled-plugins/reviewer/skills/doc-review.ts +1 -1
- package/src/bundled-plugins/reviewer/skills/plan-review.ts +1 -1
- package/src/plugin/types.ts +0 -10
- package/src/run/channel-session-factory.ts +3 -1
- package/src/run/index.ts +29 -29
- package/src/run/subagent-coalescing.ts +24 -0
- package/src/server/index.ts +4 -1
- package/src/skills/typeclaw-channel-github/SKILL.md +6 -6
- package/src/skills/typeclaw-channel-instagram/SKILL.md +1 -1
- package/src/skills/typeclaw-channel-kakaotalk/SKILL.md +1 -1
- package/src/skills/typeclaw-channel-line/SKILL.md +1 -1
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
import { walkHtmlLexically } from './html-lexical'
|
|
2
|
+
import { applyReadabilityDetailed, type ReadabilityDetailedResult } from './strategies/readability'
|
|
3
|
+
import type { HtmlExtractionAttempt } from './types'
|
|
4
|
+
|
|
5
|
+
export type AdaptiveHtmlResult =
|
|
6
|
+
| { kind: 'success'; output: string; attempts: HtmlExtractionAttempt[] }
|
|
7
|
+
| { kind: 'error'; message: string; attempts: HtmlExtractionAttempt[] }
|
|
8
|
+
|
|
9
|
+
type AdaptiveHtmlDependencies = {
|
|
10
|
+
applyReadabilityDetailed: typeof applyReadabilityDetailed
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export const MAX_ADAPTIVE_HTML_SCAN_BYTES = 5 * 1024 * 1024
|
|
14
|
+
export const MAX_ADAPTIVE_HTML_TAG_COUNT = 20_000
|
|
15
|
+
export const MAX_ADAPTIVE_HTML_NESTING_DEPTH = 256
|
|
16
|
+
export const MAX_ADAPTIVE_HTML_ATTRIBUTES_PER_TAG = 1_024
|
|
17
|
+
export const MAX_ADAPTIVE_HTML_START_TAG_CHARS = 64 * 1024
|
|
18
|
+
|
|
19
|
+
const MAX_JSON_LD_SCAN_BYTES = 1_000_000
|
|
20
|
+
const MAX_JSON_LD_BLOCKS = 32
|
|
21
|
+
const MAX_JSON_LD_BLOCK_BYTES = 256_000
|
|
22
|
+
const MAX_JSON_LD_VALUE_CHARS = 200_000
|
|
23
|
+
const ARTICLE_TYPES = new Set(['Article', 'NewsArticle', 'BlogPosting'])
|
|
24
|
+
const EMPTY_READABILITY_OUTPUT = 'Readability extracted no content from this page.'
|
|
25
|
+
const COMPLEXITY_LIMIT_REASON = 'adaptive HTML complexity limit exceeded'
|
|
26
|
+
const MALFORMED_HTML_REASON = 'adaptive HTML structure could not be validated'
|
|
27
|
+
// A bounded character floor accepts substantial script-free fallback while keeping short chrome fail-closed.
|
|
28
|
+
const SUBSTANTIAL_VISIBLE_TEXT_CHARACTERS = 512
|
|
29
|
+
const ENCODER = new TextEncoder()
|
|
30
|
+
|
|
31
|
+
export async function extractAdaptiveHtml(
|
|
32
|
+
html: string,
|
|
33
|
+
url: string,
|
|
34
|
+
dependencies: AdaptiveHtmlDependencies = { applyReadabilityDetailed },
|
|
35
|
+
): Promise<AdaptiveHtmlResult> {
|
|
36
|
+
const attempts: HtmlExtractionAttempt[] = []
|
|
37
|
+
|
|
38
|
+
const complexity = validateAdaptiveHtmlComplexity(html)
|
|
39
|
+
if (!complexity.valid) {
|
|
40
|
+
attempts.push({ route: 'readability', outcome: 'error', reason: complexity.reason })
|
|
41
|
+
} else {
|
|
42
|
+
try {
|
|
43
|
+
const readability = await dependencies.applyReadabilityDetailed(html, url)
|
|
44
|
+
const verdict = classifyReadability(readability)
|
|
45
|
+
if (verdict.useful) {
|
|
46
|
+
attempts.push({ route: 'readability', outcome: 'success', reason: verdict.reason })
|
|
47
|
+
return { kind: 'success', output: readability.output, attempts }
|
|
48
|
+
}
|
|
49
|
+
attempts.push({ route: 'readability', outcome: 'unusable', reason: verdict.reason })
|
|
50
|
+
} catch {
|
|
51
|
+
attempts.push({ route: 'readability', outcome: 'error', reason: 'Readability extraction failed' })
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
try {
|
|
56
|
+
const output = extractJsonLdArticle(html)
|
|
57
|
+
if (output !== null) {
|
|
58
|
+
attempts.push({ route: 'json_ld', outcome: 'success', reason: 'extracted schema.org article content' })
|
|
59
|
+
return { kind: 'success', output, attempts }
|
|
60
|
+
}
|
|
61
|
+
attempts.push({ route: 'json_ld', outcome: 'unusable', reason: 'no schema.org article content found' })
|
|
62
|
+
} catch {
|
|
63
|
+
attempts.push({ route: 'json_ld', outcome: 'error', reason: 'schema.org extraction failed' })
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const extractorFailed = attempts.some(({ outcome }) => outcome === 'error')
|
|
67
|
+
return {
|
|
68
|
+
kind: 'error',
|
|
69
|
+
message: extractorFailed
|
|
70
|
+
? 'An HTML extractor failed, and no public article content was recovered. Browser retrieval may be needed.'
|
|
71
|
+
: 'Fetched HTML did not contain public article content. The page may require client-side rendering or access controls; browser retrieval may be needed.',
|
|
72
|
+
attempts,
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export type AdaptiveHtmlComplexityResult = { valid: true } | { valid: false; reason: string }
|
|
77
|
+
|
|
78
|
+
export function validateAdaptiveHtmlComplexity(html: string): AdaptiveHtmlComplexityResult {
|
|
79
|
+
if (byteLength(html) > MAX_ADAPTIVE_HTML_SCAN_BYTES) return complexityFailure()
|
|
80
|
+
const result = walkHtmlLexically(html, {
|
|
81
|
+
maxTags: MAX_ADAPTIVE_HTML_TAG_COUNT,
|
|
82
|
+
maxDepth: MAX_ADAPTIVE_HTML_NESTING_DEPTH,
|
|
83
|
+
maxAttributesPerTag: MAX_ADAPTIVE_HTML_ATTRIBUTES_PER_TAG,
|
|
84
|
+
maxStartTagChars: MAX_ADAPTIVE_HTML_START_TAG_CHARS,
|
|
85
|
+
})
|
|
86
|
+
if (result.valid) return result
|
|
87
|
+
return result.failure === 'limit' ? complexityFailure() : { valid: false, reason: MALFORMED_HTML_REASON }
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export function scanJsonLdScripts(html: string): string[] {
|
|
91
|
+
if (byteLength(html) > MAX_ADAPTIVE_HTML_SCAN_BYTES) return []
|
|
92
|
+
const payloadCharacterLimit = sliceByBytes(html, MAX_JSON_LD_SCAN_BYTES).length
|
|
93
|
+
const blocks: string[] = []
|
|
94
|
+
let embeddedStylesheetSeen = false
|
|
95
|
+
let inspected = 0
|
|
96
|
+
const result = walkHtmlLexically(html, {
|
|
97
|
+
maxTags: MAX_ADAPTIVE_HTML_TAG_COUNT,
|
|
98
|
+
maxDepth: MAX_ADAPTIVE_HTML_NESTING_DEPTH,
|
|
99
|
+
maxAttributesPerTag: MAX_ADAPTIVE_HTML_ATTRIBUTES_PER_TAG,
|
|
100
|
+
maxStartTagChars: MAX_ADAPTIVE_HTML_START_TAG_CHARS,
|
|
101
|
+
onStartTag: ({ name }) => {
|
|
102
|
+
if (name === 'style') embeddedStylesheetSeen = true
|
|
103
|
+
},
|
|
104
|
+
onRawElement: (element) => {
|
|
105
|
+
if (element.name === 'style') return
|
|
106
|
+
if (
|
|
107
|
+
element.name !== 'script' ||
|
|
108
|
+
element.metadataSuppressed ||
|
|
109
|
+
element.typeAttribute !== 'application/ld+json' ||
|
|
110
|
+
element.bodyEnd > payloadCharacterLimit ||
|
|
111
|
+
inspected >= MAX_JSON_LD_BLOCKS
|
|
112
|
+
) {
|
|
113
|
+
return
|
|
114
|
+
}
|
|
115
|
+
inspected++
|
|
116
|
+
const source = html.slice(element.bodyStart, element.bodyEnd).trim()
|
|
117
|
+
if (source && byteLength(source) <= MAX_JSON_LD_BLOCK_BYTES) blocks.push(source)
|
|
118
|
+
},
|
|
119
|
+
})
|
|
120
|
+
return result.valid && !embeddedStylesheetSeen ? blocks : []
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function classifyReadability(result: ReadabilityDetailedResult): { useful: boolean; reason: string } {
|
|
124
|
+
if (result.output === EMPTY_READABILITY_OUTPUT)
|
|
125
|
+
return { useful: false, reason: 'Readability found no visible content' }
|
|
126
|
+
|
|
127
|
+
const hasClearContent =
|
|
128
|
+
result.parsedArticle &&
|
|
129
|
+
(result.hasSemanticContentContainer || result.hasArticleContainer || result.proseElementCount > 0)
|
|
130
|
+
if (hasClearContent) {
|
|
131
|
+
return { useful: true, reason: 'Readability identified article content' }
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
if (result.visibleTextCharacterCount >= SUBSTANTIAL_VISIBLE_TEXT_CHARACTERS) {
|
|
135
|
+
return { useful: true, reason: 'generic body fallback contained substantial visible content' }
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const scriptRenderedShell =
|
|
139
|
+
!result.hasSemanticContentContainer &&
|
|
140
|
+
result.scriptCount >= 2 &&
|
|
141
|
+
(result.structuredContentElementCount > 0 || result.scriptCount >= 3)
|
|
142
|
+
|
|
143
|
+
if (scriptRenderedShell) {
|
|
144
|
+
return { useful: false, reason: 'generic body fallback looked like a script-rendered shell' }
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
if (result.parsedArticle && result.structuredContentElementCount > 0) {
|
|
148
|
+
return { useful: true, reason: 'Readability identified article content' }
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
if (result.parsedArticle) {
|
|
152
|
+
return { useful: false, reason: 'Readability identified only unstructured page chrome' }
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
return { useful: false, reason: 'generic body fallback contained only short visible content' }
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function complexityFailure(): AdaptiveHtmlComplexityResult {
|
|
159
|
+
return { valid: false, reason: COMPLEXITY_LIMIT_REASON }
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function extractJsonLdArticle(html: string): string | null {
|
|
163
|
+
for (const source of scanJsonLdScripts(html)) {
|
|
164
|
+
let parsed: unknown
|
|
165
|
+
try {
|
|
166
|
+
parsed = JSON.parse(source)
|
|
167
|
+
} catch {
|
|
168
|
+
continue
|
|
169
|
+
}
|
|
170
|
+
for (const candidate of articleCandidates(parsed)) {
|
|
171
|
+
const headline = plainText(candidate.headline)
|
|
172
|
+
const body = plainText(candidate.articleBody) || plainText(candidate.description)
|
|
173
|
+
if (!body) continue
|
|
174
|
+
return headline ? `# ${escapeMarkdownHeading(headline)}\n\n${body}` : body
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
return null
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function articleCandidates(value: unknown, depth = 0, inheritedContext?: unknown): Array<Record<string, unknown>> {
|
|
181
|
+
if (depth > 4) return []
|
|
182
|
+
if (Array.isArray(value)) return value.flatMap((item) => articleCandidates(item, depth + 1, inheritedContext))
|
|
183
|
+
if (!isRecord(value)) return []
|
|
184
|
+
|
|
185
|
+
const context = Object.hasOwn(value, '@context') ? value['@context'] : inheritedContext
|
|
186
|
+
const candidates = isArticleType(value['@type'], context) ? [value] : []
|
|
187
|
+
const graph = value['@graph']
|
|
188
|
+
if (graph !== undefined) candidates.push(...articleCandidates(graph, depth + 1, context))
|
|
189
|
+
return candidates
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function isArticleType(value: unknown, context: unknown): boolean {
|
|
193
|
+
const values = Array.isArray(value) ? value : [value]
|
|
194
|
+
return values.some((item) => {
|
|
195
|
+
if (typeof item !== 'string') return false
|
|
196
|
+
const absoluteSchemaType = schemaOrgName(item)
|
|
197
|
+
if (absoluteSchemaType !== null) return ARTICLE_TYPES.has(absoluteSchemaType)
|
|
198
|
+
if (context === undefined) return ARTICLE_TYPES.has(item)
|
|
199
|
+
const resolved = resolveContextType(item, context)
|
|
200
|
+
return resolved !== null && ARTICLE_TYPES.has(resolved)
|
|
201
|
+
})
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
function schemaOrgName(value: string): string | null {
|
|
205
|
+
const match = /^https?:\/\/schema\.org\/(?:[^/#]+[/#])?([^/#]+)$/iu.exec(value)
|
|
206
|
+
return match?.[1] ?? null
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
function resolveContextType(type: string, context: unknown, depth = 0): string | null {
|
|
210
|
+
return resolveContextDefinition(type, context, depth).value
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function resolveContextDefinition(
|
|
214
|
+
type: string,
|
|
215
|
+
context: unknown,
|
|
216
|
+
depth: number,
|
|
217
|
+
): { defined: boolean; value: string | null } {
|
|
218
|
+
if (depth > 4) return { defined: false, value: null }
|
|
219
|
+
if (context === null) return { defined: true, value: null }
|
|
220
|
+
if (typeof context === 'string') {
|
|
221
|
+
return { defined: true, value: isSchemaOrgBase(context) && ARTICLE_TYPES.has(type) ? type : null }
|
|
222
|
+
}
|
|
223
|
+
if (Array.isArray(context)) {
|
|
224
|
+
for (let index = context.length - 1; index >= 0; index--) {
|
|
225
|
+
const resolution = resolveContextDefinition(type, context[index], depth + 1)
|
|
226
|
+
if (resolution.defined) return resolution
|
|
227
|
+
}
|
|
228
|
+
return { defined: false, value: null }
|
|
229
|
+
}
|
|
230
|
+
if (!isRecord(context)) return { defined: false, value: null }
|
|
231
|
+
|
|
232
|
+
const termDefined = Object.hasOwn(context, type)
|
|
233
|
+
const term = context[type]
|
|
234
|
+
const termId = typeof term === 'string' ? term : isRecord(term) ? term['@id'] : undefined
|
|
235
|
+
if (termDefined) {
|
|
236
|
+
const absolute = typeof termId === 'string' ? schemaOrgName(termId) : null
|
|
237
|
+
return { defined: true, value: absolute }
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const separator = type.indexOf(':')
|
|
241
|
+
if (separator > 0) {
|
|
242
|
+
const prefixName = type.slice(0, separator)
|
|
243
|
+
if (!Object.hasOwn(context, prefixName)) return { defined: false, value: null }
|
|
244
|
+
const prefix = context[prefixName]
|
|
245
|
+
const prefixId = typeof prefix === 'string' ? prefix : isRecord(prefix) ? prefix['@id'] : undefined
|
|
246
|
+
return {
|
|
247
|
+
defined: true,
|
|
248
|
+
value: typeof prefixId === 'string' && isSchemaOrgBase(prefixId) ? type.slice(separator + 1) : null,
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
if (!Object.hasOwn(context, '@vocab')) return { defined: false, value: null }
|
|
253
|
+
const vocabulary = context['@vocab']
|
|
254
|
+
return {
|
|
255
|
+
defined: true,
|
|
256
|
+
value: typeof vocabulary === 'string' && isSchemaOrgBase(vocabulary) ? type : null,
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
function isSchemaOrgBase(value: string): boolean {
|
|
261
|
+
return /^https?:\/\/schema\.org\/?$/iu.test(value)
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
function plainText(value: unknown): string {
|
|
265
|
+
if (typeof value !== 'string') return ''
|
|
266
|
+
return stripTags(value.slice(0, MAX_JSON_LD_VALUE_CHARS))
|
|
267
|
+
.replace(/&(?:nbsp|#160);/giu, ' ')
|
|
268
|
+
.replace(/&/giu, '&')
|
|
269
|
+
.replace(/</giu, '<')
|
|
270
|
+
.replace(/>/giu, '>')
|
|
271
|
+
.replace(/"/giu, '"')
|
|
272
|
+
.replace(/'|'/giu, "'")
|
|
273
|
+
.replace(/\s+/gu, ' ')
|
|
274
|
+
.trim()
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
function stripTags(value: string): string {
|
|
278
|
+
const output: string[] = []
|
|
279
|
+
let copiedThrough = 0
|
|
280
|
+
let cursor = 0
|
|
281
|
+
while (cursor < value.length) {
|
|
282
|
+
if (value[cursor] !== '<') {
|
|
283
|
+
cursor++
|
|
284
|
+
continue
|
|
285
|
+
}
|
|
286
|
+
output.push(value.slice(copiedThrough, cursor))
|
|
287
|
+
const candidateStart = cursor
|
|
288
|
+
cursor++
|
|
289
|
+
while (cursor < value.length && value[cursor] !== '>') cursor++
|
|
290
|
+
if (cursor >= value.length) {
|
|
291
|
+
output.push(value.slice(candidateStart))
|
|
292
|
+
copiedThrough = value.length
|
|
293
|
+
break
|
|
294
|
+
}
|
|
295
|
+
output.push(' ')
|
|
296
|
+
cursor++
|
|
297
|
+
copiedThrough = cursor
|
|
298
|
+
}
|
|
299
|
+
if (copiedThrough < value.length) output.push(value.slice(copiedThrough))
|
|
300
|
+
return output.join('')
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
function escapeMarkdownHeading(value: string): string {
|
|
304
|
+
return value.replace(/([\\`*_[\]{}()#+.!|>-])/gu, '\\$1')
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
308
|
+
return typeof value === 'object' && value !== null
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
function byteLength(value: string): number {
|
|
312
|
+
return ENCODER.encode(value).byteLength
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
function sliceByBytes(value: string, maxBytes: number): string {
|
|
316
|
+
const bytes = ENCODER.encode(value)
|
|
317
|
+
if (bytes.byteLength <= maxBytes) return value
|
|
318
|
+
return new TextDecoder('utf-8', { fatal: false }).decode(bytes.slice(0, maxBytes))
|
|
319
|
+
}
|