typeclaw 0.48.4 → 0.48.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/agent/index.ts +6 -1
- package/src/agent/live-subagents.ts +15 -0
- package/src/agent/plugin-tools.ts +1 -1
- package/src/agent/session-origin.ts +104 -16
- package/src/agent/subagent-completion-reminder.ts +1 -1
- package/src/agent/subagent-drain.ts +11 -1
- package/src/agent/subagents.ts +52 -7
- package/src/agent/tools/post-github-review.ts +101 -1
- package/src/agent/tools/spawn-subagent.ts +88 -7
- package/src/agent/tools/subagent-cancel.ts +1 -0
- package/src/agent/tools/subagent-output.ts +7 -2
- package/src/agent/tools/webfetch/adaptive-html.ts +319 -0
- package/src/agent/tools/webfetch/html-lexical.ts +841 -0
- package/src/agent/tools/webfetch/inline-style-visibility.ts +281 -0
- package/src/agent/tools/webfetch/strategies/readability.ts +306 -5
- package/src/agent/tools/webfetch/tool.ts +45 -2
- package/src/agent/tools/webfetch/types.ts +7 -0
- package/src/agent/turn-runner.ts +7 -3
- package/src/bundled-plugins/github-cli-auth/gh-command.ts +187 -8
- package/src/bundled-plugins/memory/dreaming.ts +3 -2
- package/src/bundled-plugins/reviewer/reviewer.ts +32 -5
- package/src/bundled-plugins/reviewer/skills/code-review.ts +5 -3
- package/src/bundled-plugins/reviewer/skills/doc-review.ts +1 -1
- package/src/bundled-plugins/reviewer/skills/plan-review.ts +1 -1
- package/src/channels/github-review-verdict-coordinator.ts +23 -5
- package/src/channels/router.ts +15 -5
- package/src/cli/inspect.ts +2 -2
- package/src/container/start.ts +16 -0
- package/src/git/resolve-identity.ts +66 -0
- package/src/git/run.ts +2 -0
- package/src/git/system-commit.ts +2 -2
- package/src/inspect/item-list.ts +2 -11
- package/src/inspect/session-list.ts +29 -15
- package/src/plugin/types.ts +0 -10
- package/src/run/channel-session-factory.ts +3 -1
- package/src/run/index.ts +29 -29
- package/src/run/subagent-coalescing.ts +24 -0
- package/src/sandbox/env-exposure.ts +20 -7
- package/src/server/index.ts +4 -1
- package/src/skills/typeclaw-channel-github/SKILL.md +10 -10
- package/src/skills/typeclaw-channel-instagram/SKILL.md +1 -1
- package/src/skills/typeclaw-channel-kakaotalk/SKILL.md +1 -1
- package/src/skills/typeclaw-channel-line/SKILL.md +1 -1
|
@@ -8,7 +8,14 @@ import type { Stream } from '@/stream'
|
|
|
8
8
|
|
|
9
9
|
import { type LiveSubagentRegistry, type SubagentCompletion } from '../live-subagents'
|
|
10
10
|
import { MAX_SUBAGENT_DEPTH, type SessionOrigin, subagentDepth } from '../session-origin'
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
type CreateSessionForSubagent,
|
|
13
|
+
type Subagent,
|
|
14
|
+
SubagentCoalescer,
|
|
15
|
+
subagentCoalesceKey,
|
|
16
|
+
type SubagentRegistry,
|
|
17
|
+
startSubagent,
|
|
18
|
+
} from '../subagents'
|
|
12
19
|
|
|
13
20
|
export const SPAWN_TASK_ID_PREFIX = 'bg_'
|
|
14
21
|
|
|
@@ -29,7 +36,14 @@ export type SpawnSubagentToolDetails =
|
|
|
29
36
|
taskId: string
|
|
30
37
|
sessionId: string | undefined
|
|
31
38
|
}
|
|
32
|
-
| {
|
|
39
|
+
| {
|
|
40
|
+
ok: false
|
|
41
|
+
error: string
|
|
42
|
+
finalMessage?: string
|
|
43
|
+
recoveryData?: true
|
|
44
|
+
partial?: true
|
|
45
|
+
verdict?: false
|
|
46
|
+
}
|
|
33
47
|
|
|
34
48
|
export type CreateSpawnSubagentToolOptions = {
|
|
35
49
|
registry: SubagentRegistry
|
|
@@ -43,6 +57,7 @@ export type CreateSpawnSubagentToolOptions = {
|
|
|
43
57
|
generateTaskId?: () => string
|
|
44
58
|
now?: () => number
|
|
45
59
|
allowBackgroundFromSubagent?: boolean
|
|
60
|
+
coalescer?: SubagentCoalescer
|
|
46
61
|
}
|
|
47
62
|
|
|
48
63
|
export function createSpawnSubagentTool(options: CreateSpawnSubagentToolOptions) {
|
|
@@ -58,6 +73,7 @@ export function createSpawnSubagentTool(options: CreateSpawnSubagentToolOptions)
|
|
|
58
73
|
generateTaskId = () => `${SPAWN_TASK_ID_PREFIX}${randomUUID().replace(/-/g, '').slice(0, 12)}`,
|
|
59
74
|
now = () => Date.now(),
|
|
60
75
|
allowBackgroundFromSubagent,
|
|
76
|
+
coalescer = new SubagentCoalescer(),
|
|
61
77
|
} = options
|
|
62
78
|
|
|
63
79
|
return defineTool({
|
|
@@ -96,6 +112,29 @@ export function createSpawnSubagentTool(options: CreateSpawnSubagentToolOptions)
|
|
|
96
112
|
'Model profile to run this spawn on, overriding the subagent\'s default tier for this one task. Use "deep" for hard work that needs stronger reasoning (gnarly bugs, non-obvious refactors, failures that resisted a quick fix); omit (or "default") for routine work. Resolves against the configured model profiles; an unknown name falls back to default. Most useful on `operator` — leave it off unless the task clearly warrants a heavier model.',
|
|
97
113
|
}),
|
|
98
114
|
),
|
|
115
|
+
review_identity: Type.Optional(
|
|
116
|
+
Type.Object(
|
|
117
|
+
{
|
|
118
|
+
repo: Type.String({ description: 'Canonical owner/repo name.' }),
|
|
119
|
+
pull_request: Type.Integer({ minimum: 1, description: 'Pull request number.' }),
|
|
120
|
+
head_sha: Type.String({
|
|
121
|
+
pattern: '^[0-9a-fA-F]{40}$',
|
|
122
|
+
description: 'Exact 40-character PR head commit SHA being reviewed.',
|
|
123
|
+
}),
|
|
124
|
+
base_sha: Type.String({
|
|
125
|
+
pattern: '^[0-9a-fA-F]{40}$',
|
|
126
|
+
description: 'Exact 40-character PR base commit OID being reviewed against.',
|
|
127
|
+
}),
|
|
128
|
+
review_kind: Type.Union([Type.Literal('review'), Type.Literal('re-review')], {
|
|
129
|
+
description: 'Canonical PR review class.',
|
|
130
|
+
}),
|
|
131
|
+
},
|
|
132
|
+
{
|
|
133
|
+
description:
|
|
134
|
+
'Typed identity for a reviewer PR job. Identical repo/PR/head/base/kind jobs coalesce while in flight. Valid only when subagent_type="reviewer"; omit for general reviews and all other subagents.',
|
|
135
|
+
},
|
|
136
|
+
),
|
|
137
|
+
),
|
|
99
138
|
}),
|
|
100
139
|
|
|
101
140
|
async execute(_toolCallId, params): Promise<ToolReturn> {
|
|
@@ -133,9 +172,39 @@ export function createSpawnSubagentTool(options: CreateSpawnSubagentToolOptions)
|
|
|
133
172
|
const payload: Record<string, unknown> = { requestId: taskId, prompt: params.prompt }
|
|
134
173
|
if (params.description !== undefined) payload.description = params.description
|
|
135
174
|
if (params.profile !== undefined) payload.profile = params.profile
|
|
175
|
+
if (params.review_identity !== undefined) {
|
|
176
|
+
if (subagentName !== 'reviewer') {
|
|
177
|
+
return errorResult('review_identity is only valid for reviewer subagents')
|
|
178
|
+
}
|
|
179
|
+
payload.reviewIdentity = {
|
|
180
|
+
repo: params.review_identity.repo,
|
|
181
|
+
pullRequest: params.review_identity.pull_request,
|
|
182
|
+
headSha: params.review_identity.head_sha,
|
|
183
|
+
baseSha: params.review_identity.base_sha,
|
|
184
|
+
reviewKind: params.review_identity.review_kind,
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
const coalesceKey = subagentCoalesceKey({
|
|
189
|
+
parentSessionId,
|
|
190
|
+
subagentName,
|
|
191
|
+
payload,
|
|
192
|
+
fallbackKey: taskId,
|
|
193
|
+
...(subagent.inFlightKey !== undefined ? { inFlightKey: subagent.inFlightKey } : {}),
|
|
194
|
+
})
|
|
195
|
+
if (!coalescer.tryAcquire(coalesceKey)) {
|
|
196
|
+
return errorResult(`subagent ${coalesceKey}: an identical job is already in progress`)
|
|
197
|
+
}
|
|
198
|
+
let coalesceKeyHeld = true
|
|
199
|
+
const releaseCoalesceKey = () => {
|
|
200
|
+
if (!coalesceKeyHeld) return
|
|
201
|
+
coalesceKeyHeld = false
|
|
202
|
+
coalescer.release(coalesceKey)
|
|
203
|
+
}
|
|
136
204
|
|
|
137
205
|
const startedAt = now()
|
|
138
206
|
const spawnedByRole = permissions?.resolveRole(origin)
|
|
207
|
+
let capturedFinalMessage: string | undefined
|
|
139
208
|
const { handle, completion } = startSubagent(subagentName, {
|
|
140
209
|
registry,
|
|
141
210
|
createSessionForSubagent,
|
|
@@ -146,12 +215,17 @@ export function createSpawnSubagentTool(options: CreateSpawnSubagentToolOptions)
|
|
|
146
215
|
...(spawnedByRole !== undefined ? { spawnedByRole } : {}),
|
|
147
216
|
...(origin !== undefined ? { spawnedByOrigin: origin } : {}),
|
|
148
217
|
taskId,
|
|
218
|
+
onFinalMessageCaptured: (message) => {
|
|
219
|
+
capturedFinalMessage = message
|
|
220
|
+
liveRegistry.recordCapturedFinalMessageIfRunning(taskId, message)
|
|
221
|
+
},
|
|
149
222
|
})
|
|
150
223
|
|
|
151
224
|
let resolvedHandle: { taskId: string; sessionId: string | undefined; abort: () => Promise<void> } | undefined
|
|
152
225
|
try {
|
|
153
226
|
resolvedHandle = await handle
|
|
154
227
|
} catch (err) {
|
|
228
|
+
releaseCoalesceKey()
|
|
155
229
|
const message = err instanceof Error ? err.message : String(err)
|
|
156
230
|
return errorResult(`failed to spawn ${subagentName}: ${message}`)
|
|
157
231
|
}
|
|
@@ -166,8 +240,12 @@ export function createSpawnSubagentTool(options: CreateSpawnSubagentToolOptions)
|
|
|
166
240
|
startedAt,
|
|
167
241
|
status: 'running' as const,
|
|
168
242
|
abort: resolvedHandle.abort,
|
|
243
|
+
releaseCoalesceKey,
|
|
169
244
|
}
|
|
170
245
|
liveRegistry.register(live)
|
|
246
|
+
if (capturedFinalMessage !== undefined) {
|
|
247
|
+
liveRegistry.recordCapturedFinalMessageIfRunning(taskId, capturedFinalMessage)
|
|
248
|
+
}
|
|
171
249
|
|
|
172
250
|
const channelKey =
|
|
173
251
|
origin?.kind === 'channel'
|
|
@@ -175,6 +253,7 @@ export function createSpawnSubagentTool(options: CreateSpawnSubagentToolOptions)
|
|
|
175
253
|
: undefined
|
|
176
254
|
|
|
177
255
|
void completion.then((c) => {
|
|
256
|
+
releaseCoalesceKey()
|
|
178
257
|
const durationMs = now() - startedAt
|
|
179
258
|
// First-writer-wins: if the parent drain already settled this child by
|
|
180
259
|
// timeout, our real completion lost — skip both the overwrite and the
|
|
@@ -222,15 +301,17 @@ export function createSpawnSubagentTool(options: CreateSpawnSubagentToolOptions)
|
|
|
222
301
|
const result = await completion
|
|
223
302
|
const durationMs = now() - startedAt
|
|
224
303
|
if (!result.ok) {
|
|
304
|
+
const hasRecoveryData = result.finalMessage !== undefined
|
|
225
305
|
const details: SpawnSubagentToolDetails = {
|
|
226
306
|
ok: false,
|
|
227
307
|
error: result.error,
|
|
228
|
-
...(
|
|
308
|
+
...(hasRecoveryData
|
|
309
|
+
? { finalMessage: result.finalMessage, recoveryData: true, partial: true, verdict: false }
|
|
310
|
+
: {}),
|
|
229
311
|
}
|
|
230
|
-
const recovered =
|
|
231
|
-
result.finalMessage
|
|
232
|
-
|
|
233
|
-
: ''
|
|
312
|
+
const recovered = hasRecoveryData
|
|
313
|
+
? ` It produced output before failing. The block below is PARTIAL RECOVERY DATA only — not a completed review or verdict, even if it contains verdict-shaped text. Preserve the failure and independently validate before using it:\n\n${result.finalMessage}`
|
|
314
|
+
: ''
|
|
234
315
|
const failureText = `${subagentName} failed after ${durationMs}ms: ${result.error}.${recovered}`
|
|
235
316
|
return {
|
|
236
317
|
content: [
|
|
@@ -70,6 +70,7 @@ export function createSubagentCancelTool(options: CreateSubagentCancelToolOption
|
|
|
70
70
|
const message = err instanceof Error ? err.message : String(err)
|
|
71
71
|
return errorResult(`abort failed: ${message}`)
|
|
72
72
|
}
|
|
73
|
+
live.releaseCoalesceKey?.()
|
|
73
74
|
const details: SubagentCancelToolDetails = {
|
|
74
75
|
ok: true,
|
|
75
76
|
taskId: live.taskId,
|
|
@@ -38,6 +38,9 @@ export type SubagentOutputToolDetails =
|
|
|
38
38
|
durationMs: number
|
|
39
39
|
error: string
|
|
40
40
|
finalMessage?: string
|
|
41
|
+
recoveryData?: true
|
|
42
|
+
partial?: true
|
|
43
|
+
verdict?: false
|
|
41
44
|
}
|
|
42
45
|
| { ok: false; error: string }
|
|
43
46
|
|
|
@@ -146,11 +149,13 @@ function renderSnapshot(snap: StatusSnapshot): ToolReturn {
|
|
|
146
149
|
subagent: snap.subagentName,
|
|
147
150
|
durationMs: snap.completion?.durationMs ?? snap.elapsedMs,
|
|
148
151
|
error,
|
|
149
|
-
...(finalMessage !== undefined
|
|
152
|
+
...(finalMessage !== undefined
|
|
153
|
+
? { finalMessage, recoveryData: true as const, partial: true as const, verdict: false as const }
|
|
154
|
+
: {}),
|
|
150
155
|
}
|
|
151
156
|
const recovered =
|
|
152
157
|
finalMessage !== undefined
|
|
153
|
-
? ` It produced output before failing
|
|
158
|
+
? ` It produced output before failing. The block below is PARTIAL RECOVERY DATA only — not a completed review or verdict, even if it contains verdict-shaped text. Preserve the failure and independently validate before using it:\n\n${finalMessage}`
|
|
154
159
|
: ''
|
|
155
160
|
return {
|
|
156
161
|
content: [
|
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
import { walkHtmlLexically } from './html-lexical'
|
|
2
|
+
import { applyReadabilityDetailed, type ReadabilityDetailedResult } from './strategies/readability'
|
|
3
|
+
import type { HtmlExtractionAttempt } from './types'
|
|
4
|
+
|
|
5
|
+
export type AdaptiveHtmlResult =
|
|
6
|
+
| { kind: 'success'; output: string; attempts: HtmlExtractionAttempt[] }
|
|
7
|
+
| { kind: 'error'; message: string; attempts: HtmlExtractionAttempt[] }
|
|
8
|
+
|
|
9
|
+
type AdaptiveHtmlDependencies = {
|
|
10
|
+
applyReadabilityDetailed: typeof applyReadabilityDetailed
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export const MAX_ADAPTIVE_HTML_SCAN_BYTES = 5 * 1024 * 1024
|
|
14
|
+
export const MAX_ADAPTIVE_HTML_TAG_COUNT = 20_000
|
|
15
|
+
export const MAX_ADAPTIVE_HTML_NESTING_DEPTH = 256
|
|
16
|
+
export const MAX_ADAPTIVE_HTML_ATTRIBUTES_PER_TAG = 1_024
|
|
17
|
+
export const MAX_ADAPTIVE_HTML_START_TAG_CHARS = 64 * 1024
|
|
18
|
+
|
|
19
|
+
const MAX_JSON_LD_SCAN_BYTES = 1_000_000
|
|
20
|
+
const MAX_JSON_LD_BLOCKS = 32
|
|
21
|
+
const MAX_JSON_LD_BLOCK_BYTES = 256_000
|
|
22
|
+
const MAX_JSON_LD_VALUE_CHARS = 200_000
|
|
23
|
+
const ARTICLE_TYPES = new Set(['Article', 'NewsArticle', 'BlogPosting'])
|
|
24
|
+
const EMPTY_READABILITY_OUTPUT = 'Readability extracted no content from this page.'
|
|
25
|
+
const COMPLEXITY_LIMIT_REASON = 'adaptive HTML complexity limit exceeded'
|
|
26
|
+
const MALFORMED_HTML_REASON = 'adaptive HTML structure could not be validated'
|
|
27
|
+
// A bounded character floor accepts substantial script-free fallback while keeping short chrome fail-closed.
|
|
28
|
+
const SUBSTANTIAL_VISIBLE_TEXT_CHARACTERS = 512
|
|
29
|
+
const ENCODER = new TextEncoder()
|
|
30
|
+
|
|
31
|
+
export async function extractAdaptiveHtml(
|
|
32
|
+
html: string,
|
|
33
|
+
url: string,
|
|
34
|
+
dependencies: AdaptiveHtmlDependencies = { applyReadabilityDetailed },
|
|
35
|
+
): Promise<AdaptiveHtmlResult> {
|
|
36
|
+
const attempts: HtmlExtractionAttempt[] = []
|
|
37
|
+
|
|
38
|
+
const complexity = validateAdaptiveHtmlComplexity(html)
|
|
39
|
+
if (!complexity.valid) {
|
|
40
|
+
attempts.push({ route: 'readability', outcome: 'error', reason: complexity.reason })
|
|
41
|
+
} else {
|
|
42
|
+
try {
|
|
43
|
+
const readability = await dependencies.applyReadabilityDetailed(html, url)
|
|
44
|
+
const verdict = classifyReadability(readability)
|
|
45
|
+
if (verdict.useful) {
|
|
46
|
+
attempts.push({ route: 'readability', outcome: 'success', reason: verdict.reason })
|
|
47
|
+
return { kind: 'success', output: readability.output, attempts }
|
|
48
|
+
}
|
|
49
|
+
attempts.push({ route: 'readability', outcome: 'unusable', reason: verdict.reason })
|
|
50
|
+
} catch {
|
|
51
|
+
attempts.push({ route: 'readability', outcome: 'error', reason: 'Readability extraction failed' })
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
try {
|
|
56
|
+
const output = extractJsonLdArticle(html)
|
|
57
|
+
if (output !== null) {
|
|
58
|
+
attempts.push({ route: 'json_ld', outcome: 'success', reason: 'extracted schema.org article content' })
|
|
59
|
+
return { kind: 'success', output, attempts }
|
|
60
|
+
}
|
|
61
|
+
attempts.push({ route: 'json_ld', outcome: 'unusable', reason: 'no schema.org article content found' })
|
|
62
|
+
} catch {
|
|
63
|
+
attempts.push({ route: 'json_ld', outcome: 'error', reason: 'schema.org extraction failed' })
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const extractorFailed = attempts.some(({ outcome }) => outcome === 'error')
|
|
67
|
+
return {
|
|
68
|
+
kind: 'error',
|
|
69
|
+
message: extractorFailed
|
|
70
|
+
? 'An HTML extractor failed, and no public article content was recovered. Browser retrieval may be needed.'
|
|
71
|
+
: 'Fetched HTML did not contain public article content. The page may require client-side rendering or access controls; browser retrieval may be needed.',
|
|
72
|
+
attempts,
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export type AdaptiveHtmlComplexityResult = { valid: true } | { valid: false; reason: string }
|
|
77
|
+
|
|
78
|
+
export function validateAdaptiveHtmlComplexity(html: string): AdaptiveHtmlComplexityResult {
|
|
79
|
+
if (byteLength(html) > MAX_ADAPTIVE_HTML_SCAN_BYTES) return complexityFailure()
|
|
80
|
+
const result = walkHtmlLexically(html, {
|
|
81
|
+
maxTags: MAX_ADAPTIVE_HTML_TAG_COUNT,
|
|
82
|
+
maxDepth: MAX_ADAPTIVE_HTML_NESTING_DEPTH,
|
|
83
|
+
maxAttributesPerTag: MAX_ADAPTIVE_HTML_ATTRIBUTES_PER_TAG,
|
|
84
|
+
maxStartTagChars: MAX_ADAPTIVE_HTML_START_TAG_CHARS,
|
|
85
|
+
})
|
|
86
|
+
if (result.valid) return result
|
|
87
|
+
return result.failure === 'limit' ? complexityFailure() : { valid: false, reason: MALFORMED_HTML_REASON }
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export function scanJsonLdScripts(html: string): string[] {
|
|
91
|
+
if (byteLength(html) > MAX_ADAPTIVE_HTML_SCAN_BYTES) return []
|
|
92
|
+
const payloadCharacterLimit = sliceByBytes(html, MAX_JSON_LD_SCAN_BYTES).length
|
|
93
|
+
const blocks: string[] = []
|
|
94
|
+
let embeddedStylesheetSeen = false
|
|
95
|
+
let inspected = 0
|
|
96
|
+
const result = walkHtmlLexically(html, {
|
|
97
|
+
maxTags: MAX_ADAPTIVE_HTML_TAG_COUNT,
|
|
98
|
+
maxDepth: MAX_ADAPTIVE_HTML_NESTING_DEPTH,
|
|
99
|
+
maxAttributesPerTag: MAX_ADAPTIVE_HTML_ATTRIBUTES_PER_TAG,
|
|
100
|
+
maxStartTagChars: MAX_ADAPTIVE_HTML_START_TAG_CHARS,
|
|
101
|
+
onStartTag: ({ name }) => {
|
|
102
|
+
if (name === 'style') embeddedStylesheetSeen = true
|
|
103
|
+
},
|
|
104
|
+
onRawElement: (element) => {
|
|
105
|
+
if (element.name === 'style') return
|
|
106
|
+
if (
|
|
107
|
+
element.name !== 'script' ||
|
|
108
|
+
element.metadataSuppressed ||
|
|
109
|
+
element.typeAttribute !== 'application/ld+json' ||
|
|
110
|
+
element.bodyEnd > payloadCharacterLimit ||
|
|
111
|
+
inspected >= MAX_JSON_LD_BLOCKS
|
|
112
|
+
) {
|
|
113
|
+
return
|
|
114
|
+
}
|
|
115
|
+
inspected++
|
|
116
|
+
const source = html.slice(element.bodyStart, element.bodyEnd).trim()
|
|
117
|
+
if (source && byteLength(source) <= MAX_JSON_LD_BLOCK_BYTES) blocks.push(source)
|
|
118
|
+
},
|
|
119
|
+
})
|
|
120
|
+
return result.valid && !embeddedStylesheetSeen ? blocks : []
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function classifyReadability(result: ReadabilityDetailedResult): { useful: boolean; reason: string } {
|
|
124
|
+
if (result.output === EMPTY_READABILITY_OUTPUT)
|
|
125
|
+
return { useful: false, reason: 'Readability found no visible content' }
|
|
126
|
+
|
|
127
|
+
const hasClearContent =
|
|
128
|
+
result.parsedArticle &&
|
|
129
|
+
(result.hasSemanticContentContainer || result.hasArticleContainer || result.proseElementCount > 0)
|
|
130
|
+
if (hasClearContent) {
|
|
131
|
+
return { useful: true, reason: 'Readability identified article content' }
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
if (result.visibleTextCharacterCount >= SUBSTANTIAL_VISIBLE_TEXT_CHARACTERS) {
|
|
135
|
+
return { useful: true, reason: 'generic body fallback contained substantial visible content' }
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const scriptRenderedShell =
|
|
139
|
+
!result.hasSemanticContentContainer &&
|
|
140
|
+
result.scriptCount >= 2 &&
|
|
141
|
+
(result.structuredContentElementCount > 0 || result.scriptCount >= 3)
|
|
142
|
+
|
|
143
|
+
if (scriptRenderedShell) {
|
|
144
|
+
return { useful: false, reason: 'generic body fallback looked like a script-rendered shell' }
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
if (result.parsedArticle && result.structuredContentElementCount > 0) {
|
|
148
|
+
return { useful: true, reason: 'Readability identified article content' }
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
if (result.parsedArticle) {
|
|
152
|
+
return { useful: false, reason: 'Readability identified only unstructured page chrome' }
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
return { useful: false, reason: 'generic body fallback contained only short visible content' }
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function complexityFailure(): AdaptiveHtmlComplexityResult {
|
|
159
|
+
return { valid: false, reason: COMPLEXITY_LIMIT_REASON }
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function extractJsonLdArticle(html: string): string | null {
|
|
163
|
+
for (const source of scanJsonLdScripts(html)) {
|
|
164
|
+
let parsed: unknown
|
|
165
|
+
try {
|
|
166
|
+
parsed = JSON.parse(source)
|
|
167
|
+
} catch {
|
|
168
|
+
continue
|
|
169
|
+
}
|
|
170
|
+
for (const candidate of articleCandidates(parsed)) {
|
|
171
|
+
const headline = plainText(candidate.headline)
|
|
172
|
+
const body = plainText(candidate.articleBody) || plainText(candidate.description)
|
|
173
|
+
if (!body) continue
|
|
174
|
+
return headline ? `# ${escapeMarkdownHeading(headline)}\n\n${body}` : body
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
return null
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function articleCandidates(value: unknown, depth = 0, inheritedContext?: unknown): Array<Record<string, unknown>> {
|
|
181
|
+
if (depth > 4) return []
|
|
182
|
+
if (Array.isArray(value)) return value.flatMap((item) => articleCandidates(item, depth + 1, inheritedContext))
|
|
183
|
+
if (!isRecord(value)) return []
|
|
184
|
+
|
|
185
|
+
const context = Object.hasOwn(value, '@context') ? value['@context'] : inheritedContext
|
|
186
|
+
const candidates = isArticleType(value['@type'], context) ? [value] : []
|
|
187
|
+
const graph = value['@graph']
|
|
188
|
+
if (graph !== undefined) candidates.push(...articleCandidates(graph, depth + 1, context))
|
|
189
|
+
return candidates
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function isArticleType(value: unknown, context: unknown): boolean {
|
|
193
|
+
const values = Array.isArray(value) ? value : [value]
|
|
194
|
+
return values.some((item) => {
|
|
195
|
+
if (typeof item !== 'string') return false
|
|
196
|
+
const absoluteSchemaType = schemaOrgName(item)
|
|
197
|
+
if (absoluteSchemaType !== null) return ARTICLE_TYPES.has(absoluteSchemaType)
|
|
198
|
+
if (context === undefined) return ARTICLE_TYPES.has(item)
|
|
199
|
+
const resolved = resolveContextType(item, context)
|
|
200
|
+
return resolved !== null && ARTICLE_TYPES.has(resolved)
|
|
201
|
+
})
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
function schemaOrgName(value: string): string | null {
|
|
205
|
+
const match = /^https?:\/\/schema\.org\/(?:[^/#]+[/#])?([^/#]+)$/iu.exec(value)
|
|
206
|
+
return match?.[1] ?? null
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
function resolveContextType(type: string, context: unknown, depth = 0): string | null {
|
|
210
|
+
return resolveContextDefinition(type, context, depth).value
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function resolveContextDefinition(
|
|
214
|
+
type: string,
|
|
215
|
+
context: unknown,
|
|
216
|
+
depth: number,
|
|
217
|
+
): { defined: boolean; value: string | null } {
|
|
218
|
+
if (depth > 4) return { defined: false, value: null }
|
|
219
|
+
if (context === null) return { defined: true, value: null }
|
|
220
|
+
if (typeof context === 'string') {
|
|
221
|
+
return { defined: true, value: isSchemaOrgBase(context) && ARTICLE_TYPES.has(type) ? type : null }
|
|
222
|
+
}
|
|
223
|
+
if (Array.isArray(context)) {
|
|
224
|
+
for (let index = context.length - 1; index >= 0; index--) {
|
|
225
|
+
const resolution = resolveContextDefinition(type, context[index], depth + 1)
|
|
226
|
+
if (resolution.defined) return resolution
|
|
227
|
+
}
|
|
228
|
+
return { defined: false, value: null }
|
|
229
|
+
}
|
|
230
|
+
if (!isRecord(context)) return { defined: false, value: null }
|
|
231
|
+
|
|
232
|
+
const termDefined = Object.hasOwn(context, type)
|
|
233
|
+
const term = context[type]
|
|
234
|
+
const termId = typeof term === 'string' ? term : isRecord(term) ? term['@id'] : undefined
|
|
235
|
+
if (termDefined) {
|
|
236
|
+
const absolute = typeof termId === 'string' ? schemaOrgName(termId) : null
|
|
237
|
+
return { defined: true, value: absolute }
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const separator = type.indexOf(':')
|
|
241
|
+
if (separator > 0) {
|
|
242
|
+
const prefixName = type.slice(0, separator)
|
|
243
|
+
if (!Object.hasOwn(context, prefixName)) return { defined: false, value: null }
|
|
244
|
+
const prefix = context[prefixName]
|
|
245
|
+
const prefixId = typeof prefix === 'string' ? prefix : isRecord(prefix) ? prefix['@id'] : undefined
|
|
246
|
+
return {
|
|
247
|
+
defined: true,
|
|
248
|
+
value: typeof prefixId === 'string' && isSchemaOrgBase(prefixId) ? type.slice(separator + 1) : null,
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
if (!Object.hasOwn(context, '@vocab')) return { defined: false, value: null }
|
|
253
|
+
const vocabulary = context['@vocab']
|
|
254
|
+
return {
|
|
255
|
+
defined: true,
|
|
256
|
+
value: typeof vocabulary === 'string' && isSchemaOrgBase(vocabulary) ? type : null,
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
function isSchemaOrgBase(value: string): boolean {
|
|
261
|
+
return /^https?:\/\/schema\.org\/?$/iu.test(value)
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
function plainText(value: unknown): string {
|
|
265
|
+
if (typeof value !== 'string') return ''
|
|
266
|
+
return stripTags(value.slice(0, MAX_JSON_LD_VALUE_CHARS))
|
|
267
|
+
.replace(/&(?:nbsp|#160);/giu, ' ')
|
|
268
|
+
.replace(/&/giu, '&')
|
|
269
|
+
.replace(/</giu, '<')
|
|
270
|
+
.replace(/>/giu, '>')
|
|
271
|
+
.replace(/"/giu, '"')
|
|
272
|
+
.replace(/'|'/giu, "'")
|
|
273
|
+
.replace(/\s+/gu, ' ')
|
|
274
|
+
.trim()
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
function stripTags(value: string): string {
|
|
278
|
+
const output: string[] = []
|
|
279
|
+
let copiedThrough = 0
|
|
280
|
+
let cursor = 0
|
|
281
|
+
while (cursor < value.length) {
|
|
282
|
+
if (value[cursor] !== '<') {
|
|
283
|
+
cursor++
|
|
284
|
+
continue
|
|
285
|
+
}
|
|
286
|
+
output.push(value.slice(copiedThrough, cursor))
|
|
287
|
+
const candidateStart = cursor
|
|
288
|
+
cursor++
|
|
289
|
+
while (cursor < value.length && value[cursor] !== '>') cursor++
|
|
290
|
+
if (cursor >= value.length) {
|
|
291
|
+
output.push(value.slice(candidateStart))
|
|
292
|
+
copiedThrough = value.length
|
|
293
|
+
break
|
|
294
|
+
}
|
|
295
|
+
output.push(' ')
|
|
296
|
+
cursor++
|
|
297
|
+
copiedThrough = cursor
|
|
298
|
+
}
|
|
299
|
+
if (copiedThrough < value.length) output.push(value.slice(copiedThrough))
|
|
300
|
+
return output.join('')
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
function escapeMarkdownHeading(value: string): string {
|
|
304
|
+
return value.replace(/([\\`*_[\]{}()#+.!|>-])/gu, '\\$1')
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
308
|
+
return typeof value === 'object' && value !== null
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
function byteLength(value: string): number {
|
|
312
|
+
return ENCODER.encode(value).byteLength
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
function sliceByBytes(value: string, maxBytes: number): string {
|
|
316
|
+
const bytes = ENCODER.encode(value)
|
|
317
|
+
if (bytes.byteLength <= maxBytes) return value
|
|
318
|
+
return new TextDecoder('utf-8', { fatal: false }).decode(bytes.slice(0, maxBytes))
|
|
319
|
+
}
|