typeclaw 0.48.3 → 0.48.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/scripts/dump-system-prompt.ts +146 -11
- package/src/agent/index.ts +6 -1
- package/src/agent/live-subagents.ts +15 -0
- package/src/agent/plugin-tools.ts +7 -0
- package/src/agent/session-origin.ts +104 -16
- package/src/agent/subagent-completion-reminder.ts +1 -1
- package/src/agent/subagent-drain.ts +11 -1
- package/src/agent/subagents.ts +52 -7
- package/src/agent/tool-file-safety.ts +14 -4
- package/src/agent/tools/runtime-notice.ts +29 -0
- package/src/agent/tools/spawn-subagent.ts +88 -7
- package/src/agent/tools/subagent-cancel.ts +1 -0
- package/src/agent/tools/subagent-output.ts +7 -2
- package/src/agent/tools/webfetch/adaptive-html.ts +319 -0
- package/src/agent/tools/webfetch/html-lexical.ts +841 -0
- package/src/agent/tools/webfetch/inline-style-visibility.ts +281 -0
- package/src/agent/tools/webfetch/strategies/readability.ts +306 -5
- package/src/agent/tools/webfetch/tool.ts +45 -2
- package/src/agent/tools/webfetch/types.ts +7 -0
- package/src/agent/turn-runner.ts +7 -3
- package/src/bundled-plugins/github-cli-auth/gh-command.ts +187 -8
- package/src/bundled-plugins/github-cli-auth/git-command.ts +497 -652
- package/src/bundled-plugins/memory/load-memory.ts +23 -27
- package/src/bundled-plugins/reviewer/reviewer.ts +32 -5
- package/src/bundled-plugins/reviewer/skills/code-review.ts +5 -3
- package/src/bundled-plugins/reviewer/skills/doc-review.ts +1 -1
- package/src/bundled-plugins/reviewer/skills/plan-review.ts +1 -1
- package/src/bundled-plugins/security/index.ts +1 -0
- package/src/bundled-plugins/security/policies/private-surface-read.ts +74 -18
- package/src/channels/router.ts +32 -47
- package/src/cli/hostd.ts +1 -1
- package/src/hostd/kakao-renewal-manager.ts +23 -11
- package/src/plugin/index.ts +1 -0
- package/src/plugin/types.ts +9 -10
- package/src/run/channel-session-factory.ts +3 -1
- package/src/run/index.ts +29 -29
- package/src/run/subagent-coalescing.ts +24 -0
- package/src/secrets/kakao-renewal.ts +123 -20
- package/src/server/index.ts +4 -1
- package/src/skills/typeclaw-channel-github/SKILL.md +6 -6
- package/src/skills/typeclaw-channel-instagram/SKILL.md +1 -1
- package/src/skills/typeclaw-channel-kakaotalk/SKILL.md +1 -1
- package/src/skills/typeclaw-channel-line/SKILL.md +1 -1
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
export type InlineStyleVisibility = 'hidden' | 'visible' | 'uncertain'
|
|
2
|
+
|
|
3
|
+
const MAX_RAW_INLINE_STYLE_CHARS = 64 * 1024
|
|
4
|
+
const MAX_CSS_NESTING_DEPTH = 32
|
|
5
|
+
const RELEVANT_PROPERTIES = new Set(['content-visibility', 'display', 'visibility', 'opacity'])
|
|
6
|
+
const CSS_NUMBER = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:e[+-]?\d+)?$/iu
|
|
7
|
+
const CSS_PERCENTAGE = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:e[+-]?\d+)?%$/iu
|
|
8
|
+
const VISIBLE_DISPLAY_VALUES = new Set([
|
|
9
|
+
'block',
|
|
10
|
+
'contents',
|
|
11
|
+
'flex',
|
|
12
|
+
'flow-root',
|
|
13
|
+
'grid',
|
|
14
|
+
'inline',
|
|
15
|
+
'inline-block',
|
|
16
|
+
'inline-flex',
|
|
17
|
+
'inline-grid',
|
|
18
|
+
'inline-table',
|
|
19
|
+
'list-item',
|
|
20
|
+
'ruby',
|
|
21
|
+
'ruby-base',
|
|
22
|
+
'ruby-base-container',
|
|
23
|
+
'ruby-text',
|
|
24
|
+
'ruby-text-container',
|
|
25
|
+
'table',
|
|
26
|
+
'table-caption',
|
|
27
|
+
'table-cell',
|
|
28
|
+
'table-column',
|
|
29
|
+
'table-column-group',
|
|
30
|
+
'table-footer-group',
|
|
31
|
+
'table-header-group',
|
|
32
|
+
'table-row',
|
|
33
|
+
'table-row-group',
|
|
34
|
+
])
|
|
35
|
+
|
|
36
|
+
type RelevantDeclaration = {
|
|
37
|
+
importance: 'normal' | 'important' | 'uncertain'
|
|
38
|
+
visibility: InlineStyleVisibility
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// The result covers only deterministic inline display, visibility, opacity, and content visibility.
|
|
42
|
+
// "Uncertain" means a relevant declaration cannot be safely classified and callers
|
|
43
|
+
// handling public extraction should suppress it just like "hidden".
|
|
44
|
+
export function classifyRawInlineStyleVisibility(style: string): InlineStyleVisibility {
|
|
45
|
+
if (style.length > MAX_RAW_INLINE_STYLE_CHARS) return 'uncertain'
|
|
46
|
+
const scanned = scanDeclarations(style)
|
|
47
|
+
if (scanned === null) return 'uncertain'
|
|
48
|
+
|
|
49
|
+
const declarations = new Map<string, RelevantDeclaration[]>()
|
|
50
|
+
for (const { rawProperty, rawValue } of scanned) {
|
|
51
|
+
const decodedProperty = decodeCssEscapes(rawProperty)
|
|
52
|
+
if (!decodedProperty.valid) {
|
|
53
|
+
if (!rawProperty.trimStart().startsWith('--')) return 'uncertain'
|
|
54
|
+
continue
|
|
55
|
+
}
|
|
56
|
+
const property = asciiFold(decodedProperty.value.trim())
|
|
57
|
+
if (!RELEVANT_PROPERTIES.has(property)) continue
|
|
58
|
+
|
|
59
|
+
const declaration = classifyDeclaration(property, rawValue)
|
|
60
|
+
const prior = declarations.get(property)
|
|
61
|
+
if (prior === undefined) declarations.set(property, [declaration])
|
|
62
|
+
else prior.push(declaration)
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
let result: InlineStyleVisibility = 'visible'
|
|
66
|
+
for (const propertyDeclarations of declarations.values()) {
|
|
67
|
+
const winner = winningDeclaration(propertyDeclarations)
|
|
68
|
+
if (winner === 'hidden') return 'hidden'
|
|
69
|
+
if (winner === 'uncertain') result = 'uncertain'
|
|
70
|
+
}
|
|
71
|
+
return result
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
type ScannedDeclaration = { rawProperty: string; rawValue: string }
|
|
75
|
+
|
|
76
|
+
function scanDeclarations(style: string): ScannedDeclaration[] | null {
|
|
77
|
+
const declarations: ScannedDeclaration[] = []
|
|
78
|
+
let declaration: string[] = []
|
|
79
|
+
let separator = -1
|
|
80
|
+
let quote: '"' | "'" | null = null
|
|
81
|
+
const blocks: string[] = []
|
|
82
|
+
let cursor = 0
|
|
83
|
+
|
|
84
|
+
const finishDeclaration = (): void => {
|
|
85
|
+
if (separator >= 0) {
|
|
86
|
+
const source = declaration.join('')
|
|
87
|
+
declarations.push({ rawProperty: source.slice(0, separator), rawValue: source.slice(separator + 1) })
|
|
88
|
+
}
|
|
89
|
+
declaration = []
|
|
90
|
+
separator = -1
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
while (cursor < style.length) {
|
|
94
|
+
const char = style[cursor]!
|
|
95
|
+
if (char === '\\') {
|
|
96
|
+
const escaped = style[cursor + 1]
|
|
97
|
+
if (escaped === undefined) return null
|
|
98
|
+
if (isCssNewline(escaped)) {
|
|
99
|
+
cursor += escaped === '\r' && style[cursor + 2] === '\n' ? 3 : 2
|
|
100
|
+
continue
|
|
101
|
+
}
|
|
102
|
+
declaration.push(char, escaped)
|
|
103
|
+
cursor += 2
|
|
104
|
+
continue
|
|
105
|
+
}
|
|
106
|
+
if (quote !== null) {
|
|
107
|
+
declaration.push(char)
|
|
108
|
+
if (char === quote) quote = null
|
|
109
|
+
cursor++
|
|
110
|
+
continue
|
|
111
|
+
}
|
|
112
|
+
if (style.startsWith('/*', cursor)) {
|
|
113
|
+
const end = style.indexOf('*/', cursor + 2)
|
|
114
|
+
if (end < 0) return null
|
|
115
|
+
declaration.push(' ')
|
|
116
|
+
cursor = end + 2
|
|
117
|
+
continue
|
|
118
|
+
}
|
|
119
|
+
if (char === '"' || char === "'") {
|
|
120
|
+
quote = char
|
|
121
|
+
declaration.push(char)
|
|
122
|
+
cursor++
|
|
123
|
+
continue
|
|
124
|
+
}
|
|
125
|
+
if (char === '(' || char === '[' || char === '{') {
|
|
126
|
+
blocks.push(char)
|
|
127
|
+
if (blocks.length > MAX_CSS_NESTING_DEPTH) return null
|
|
128
|
+
declaration.push(char)
|
|
129
|
+
cursor++
|
|
130
|
+
continue
|
|
131
|
+
}
|
|
132
|
+
if (char === ')' || char === ']' || char === '}') {
|
|
133
|
+
if (blocks.pop() !== matchingBlockStart(char)) return null
|
|
134
|
+
declaration.push(char)
|
|
135
|
+
cursor++
|
|
136
|
+
continue
|
|
137
|
+
}
|
|
138
|
+
if (blocks.length === 0 && char === ':' && separator < 0) separator = declaration.length
|
|
139
|
+
if (blocks.length === 0 && char === ';') finishDeclaration()
|
|
140
|
+
else declaration.push(char)
|
|
141
|
+
cursor++
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
if (quote !== null || blocks.length !== 0) return null
|
|
145
|
+
finishDeclaration()
|
|
146
|
+
return declarations
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function classifyDeclaration(property: string, rawValue: string): RelevantDeclaration {
|
|
150
|
+
const decoded = decodeCssEscapes(rawValue)
|
|
151
|
+
if (!decoded.valid) return { importance: 'normal', visibility: 'uncertain' }
|
|
152
|
+
|
|
153
|
+
const important = extractImportance(decoded.value)
|
|
154
|
+
if (important.importance === 'uncertain') return { importance: 'uncertain', visibility: 'uncertain' }
|
|
155
|
+
const value = asciiFold(important.value.trim())
|
|
156
|
+
|
|
157
|
+
return {
|
|
158
|
+
importance: important.importance,
|
|
159
|
+
visibility:
|
|
160
|
+
property === 'opacity'
|
|
161
|
+
? classifyOpacity(value)
|
|
162
|
+
: property === 'content-visibility'
|
|
163
|
+
? classifyContentVisibility(value)
|
|
164
|
+
: property === 'display'
|
|
165
|
+
? classifyDisplay(value)
|
|
166
|
+
: classifyVisibility(value),
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function classifyContentVisibility(value: string): InlineStyleVisibility {
|
|
171
|
+
if (value === 'hidden') return 'hidden'
|
|
172
|
+
if (value === 'visible' || value === 'auto' || value === 'initial') return 'visible'
|
|
173
|
+
return 'uncertain'
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
function winningDeclaration(declarations: RelevantDeclaration[]): InlineStyleVisibility {
|
|
177
|
+
if (declarations.some(({ importance }) => importance === 'uncertain')) return 'uncertain'
|
|
178
|
+
const important = declarations.filter(({ importance }) => importance === 'important')
|
|
179
|
+
return (important.length > 0 ? important : declarations).at(-1)?.visibility ?? 'visible'
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function classifyDisplay(value: string): InlineStyleVisibility {
|
|
183
|
+
if (value === 'none') return 'hidden'
|
|
184
|
+
if (value === 'initial') return 'visible'
|
|
185
|
+
return VISIBLE_DISPLAY_VALUES.has(value) ? 'visible' : 'uncertain'
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
function classifyVisibility(value: string): InlineStyleVisibility {
|
|
189
|
+
if (value === 'hidden' || value === 'collapse') return 'hidden'
|
|
190
|
+
if (value === 'visible' || value === 'initial') return 'visible'
|
|
191
|
+
return 'uncertain'
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function classifyOpacity(value: string): InlineStyleVisibility {
|
|
195
|
+
if (value === 'initial') return 'visible'
|
|
196
|
+
const simpleCalc = /^calc\(\s*([+-]?(?:\d+\.?\d*|\.\d+)(?:e[+-]?\d+)?%?)\s*\)$/iu.exec(value)
|
|
197
|
+
const numeric = simpleCalc?.[1] ?? value
|
|
198
|
+
if (!CSS_NUMBER.test(numeric) && !CSS_PERCENTAGE.test(numeric)) return 'uncertain'
|
|
199
|
+
const number = Number(numeric.endsWith('%') ? numeric.slice(0, -1) : numeric)
|
|
200
|
+
if (!Number.isFinite(number)) return 'uncertain'
|
|
201
|
+
if (simpleCalc !== null) return number === 0 ? 'hidden' : 'uncertain'
|
|
202
|
+
return number <= 0 ? 'hidden' : 'visible'
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function extractImportance(value: string): {
|
|
206
|
+
importance: 'normal' | 'important' | 'uncertain'
|
|
207
|
+
value: string
|
|
208
|
+
} {
|
|
209
|
+
const marker = /!\s*important\s*$/iu.exec(value)
|
|
210
|
+
if (marker !== null) {
|
|
211
|
+
const before = value.slice(0, marker.index)
|
|
212
|
+
return { importance: before.includes('!') ? 'uncertain' : 'important', value: before }
|
|
213
|
+
}
|
|
214
|
+
return value.includes('!') ? { importance: 'uncertain', value } : { importance: 'normal', value }
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function decodeCssEscapes(value: string): { valid: true; value: string } | { valid: false } {
|
|
218
|
+
let decoded = ''
|
|
219
|
+
let cursor = 0
|
|
220
|
+
while (cursor < value.length) {
|
|
221
|
+
const char = value[cursor]!
|
|
222
|
+
if (char !== '\\') {
|
|
223
|
+
decoded += char
|
|
224
|
+
cursor++
|
|
225
|
+
continue
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
cursor++
|
|
229
|
+
const escaped = value[cursor]
|
|
230
|
+
if (escaped === undefined) return { valid: false }
|
|
231
|
+
if (isCssNewline(escaped)) {
|
|
232
|
+
cursor += escaped === '\r' && value[cursor + 1] === '\n' ? 2 : 1
|
|
233
|
+
continue
|
|
234
|
+
}
|
|
235
|
+
if (!isAsciiHexDigit(escaped)) {
|
|
236
|
+
decoded += escaped
|
|
237
|
+
cursor++
|
|
238
|
+
continue
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
const hexStart = cursor
|
|
242
|
+
while (cursor < value.length && cursor - hexStart < 6 && isAsciiHexDigit(value[cursor]!)) cursor++
|
|
243
|
+
const codePoint = Number.parseInt(value.slice(hexStart, cursor), 16)
|
|
244
|
+
decoded +=
|
|
245
|
+
codePoint === 0 || codePoint > 0x10ffff || (codePoint >= 0xd800 && codePoint <= 0xdfff)
|
|
246
|
+
? '\uFFFD'
|
|
247
|
+
: String.fromCodePoint(codePoint)
|
|
248
|
+
if (isCssWhitespace(value[cursor])) {
|
|
249
|
+
if (value[cursor] === '\r' && value[cursor + 1] === '\n') cursor++
|
|
250
|
+
cursor++
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
return { valid: true, value: decoded }
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
function isAsciiHexDigit(char: string): boolean {
|
|
257
|
+
const code = char.charCodeAt(0)
|
|
258
|
+
return (code >= 48 && code <= 57) || (code >= 65 && code <= 70) || (code >= 97 && code <= 102)
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
function isCssWhitespace(char: string | undefined): boolean {
|
|
262
|
+
return char === '\t' || char === '\n' || char === '\f' || char === '\r' || char === ' '
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
function isCssNewline(char: string): boolean {
|
|
266
|
+
return char === '\n' || char === '\r' || char === '\f'
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
function matchingBlockStart(close: ')' | ']' | '}'): '(' | '[' | '{' {
|
|
270
|
+
if (close === ')') return '('
|
|
271
|
+
return close === ']' ? '[' : '{'
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
function asciiFold(value: string): string {
|
|
275
|
+
let folded = ''
|
|
276
|
+
for (const char of value) {
|
|
277
|
+
const code = char.charCodeAt(0)
|
|
278
|
+
folded += code >= 65 && code <= 90 ? String.fromCharCode(code + 32) : char
|
|
279
|
+
}
|
|
280
|
+
return folded
|
|
281
|
+
}
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import type { Readability as ReadabilityClass } from '@mozilla/readability'
|
|
2
2
|
import type TurndownService from 'turndown'
|
|
3
3
|
|
|
4
|
+
import { walkHtmlLexically } from '../html-lexical'
|
|
5
|
+
import { classifyRawInlineStyleVisibility } from '../inline-style-visibility'
|
|
6
|
+
|
|
4
7
|
// Perf: jsdom (+readability+turndown) costs ~40-60MB RSS at import time. Keep it
|
|
5
8
|
// lazy so idle agents that never call web_fetch don't pay for it. Do NOT hoist to
|
|
6
9
|
// a top-level import.
|
|
@@ -12,6 +15,12 @@ type ReadabilityDeps = {
|
|
|
12
15
|
|
|
13
16
|
let depsPromise: Promise<ReadabilityDeps> | undefined
|
|
14
17
|
|
|
18
|
+
const MAX_EMBEDDED_STYLE_CHARS = 256 * 1024
|
|
19
|
+
const MAX_EMBEDDED_STYLE_RULES = 1_024
|
|
20
|
+
const MAX_COMPUTED_STYLE_WORK = 2_000_000
|
|
21
|
+
const MAX_SELECTOR_CHARS = 2_048
|
|
22
|
+
const MAX_RAW_CSS_SEGMENT_CHARS = 16 * 1024
|
|
23
|
+
|
|
15
24
|
async function loadDeps(): Promise<ReadabilityDeps> {
|
|
16
25
|
depsPromise ??= (async () => {
|
|
17
26
|
const [{ Readability }, { JSDOM }, { default: TurndownCtor }] = await Promise.all([
|
|
@@ -34,20 +43,312 @@ async function loadDeps(): Promise<ReadabilityDeps> {
|
|
|
34
43
|
|
|
35
44
|
type ReadabilityDocument = ConstructorParameters<typeof ReadabilityClass>[0]
|
|
36
45
|
|
|
46
|
+
export type ReadabilityDetailedResult = {
|
|
47
|
+
output: string
|
|
48
|
+
parsedArticle: boolean
|
|
49
|
+
usedBodyFallback: boolean
|
|
50
|
+
hasSemanticContentContainer: boolean
|
|
51
|
+
hasArticleContainer: boolean
|
|
52
|
+
proseElementCount: number
|
|
53
|
+
structuredContentElementCount: number
|
|
54
|
+
scriptCount: number
|
|
55
|
+
visibleTextCharacterCount: number
|
|
56
|
+
}
|
|
57
|
+
|
|
37
58
|
export async function applyReadability(html: string, url: string): Promise<string> {
|
|
38
|
-
|
|
59
|
+
return (await applyReadabilityDetailed(html, url)).output
|
|
60
|
+
}
|
|
39
61
|
|
|
62
|
+
export async function applyReadabilityDetailed(html: string, url: string): Promise<ReadabilityDetailedResult> {
|
|
63
|
+
preflightEmbeddedStylesheets(html)
|
|
64
|
+
const { Readability, JSDOM, turndown } = await loadDeps()
|
|
40
65
|
const dom = new JSDOM(html, { url })
|
|
41
|
-
const
|
|
66
|
+
const sourceDocument = dom.window.document
|
|
67
|
+
const computedStyle = buildComputedStyleResolver(sourceDocument, (element) => dom.window.getComputedStyle(element))
|
|
68
|
+
pruneHiddenContent(sourceDocument, computedStyle)
|
|
69
|
+
const document = sourceDocument.cloneNode(true) as Document & ReadabilityDocument
|
|
70
|
+
const scriptCount = document.querySelectorAll('script').length
|
|
71
|
+
document.querySelectorAll('script, style, template, noscript, noframes').forEach((element) => element.remove())
|
|
72
|
+
|
|
73
|
+
const hasSemanticContentContainer = document.querySelector('article, main, [role="main"]') !== null
|
|
74
|
+
const hasArticleContainer = document.querySelector('article') !== null
|
|
75
|
+
const proseElementCount = document.querySelectorAll('p, blockquote, pre').length
|
|
76
|
+
const structuredContentElementCount = document.querySelectorAll('h1, h2, h3, h4, h5, h6, dl, table').length
|
|
77
|
+
const visibleTextCharacterCount = countVisibleTextCharacters(document.body)
|
|
78
|
+
const fallbackSource = document.body?.innerHTML ?? ''
|
|
42
79
|
const article = new Readability(document).parse()
|
|
43
80
|
|
|
44
|
-
const source = article?.content?.trim() ? article.content :
|
|
81
|
+
const source = article?.content?.trim() ? article.content : fallbackSource
|
|
45
82
|
const markdown = turndown.turndown(source).trim()
|
|
83
|
+
const output = renderOutput(markdown, article?.title)
|
|
84
|
+
|
|
85
|
+
return {
|
|
86
|
+
output,
|
|
87
|
+
parsedArticle: article?.content?.trim() !== undefined && article.content.trim().length > 0,
|
|
88
|
+
usedBodyFallback: !article?.content?.trim(),
|
|
89
|
+
hasSemanticContentContainer,
|
|
90
|
+
hasArticleContainer,
|
|
91
|
+
proseElementCount,
|
|
92
|
+
structuredContentElementCount,
|
|
93
|
+
scriptCount,
|
|
94
|
+
visibleTextCharacterCount,
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function preflightEmbeddedStylesheets(html: string): void {
|
|
99
|
+
const sources: string[] = []
|
|
100
|
+
let elementCount = 0
|
|
101
|
+
let styleStartCount = 0
|
|
102
|
+
const result = walkHtmlLexically(html, {
|
|
103
|
+
onStartTag: ({ name }) => {
|
|
104
|
+
elementCount++
|
|
105
|
+
if (name === 'style') styleStartCount++
|
|
106
|
+
},
|
|
107
|
+
onRawElement: (element) => {
|
|
108
|
+
if (element.name === 'style') sources.push(html.slice(element.bodyStart, element.bodyEnd))
|
|
109
|
+
},
|
|
110
|
+
})
|
|
111
|
+
if (!result.valid || styleStartCount === 0) return
|
|
112
|
+
if (sources.length !== styleStartCount) throw new Error('Embedded stylesheet complexity limit exceeded')
|
|
113
|
+
|
|
114
|
+
let characters = 0
|
|
115
|
+
let rules = 0
|
|
116
|
+
let selectorComponents = 0
|
|
117
|
+
for (const source of sources) {
|
|
118
|
+
characters += source.length
|
|
119
|
+
if (characters > MAX_EMBEDDED_STYLE_CHARS) throw new Error('Embedded stylesheet complexity limit exceeded')
|
|
120
|
+
const counted = countRawCssRules(source)
|
|
121
|
+
rules += counted.rules
|
|
122
|
+
selectorComponents += counted.selectorComponents
|
|
123
|
+
if (rules > MAX_EMBEDDED_STYLE_RULES) throw new Error('Embedded stylesheet complexity limit exceeded')
|
|
124
|
+
}
|
|
125
|
+
if (selectorComponents * elementCount > MAX_COMPUTED_STYLE_WORK) {
|
|
126
|
+
throw new Error('Embedded stylesheet complexity limit exceeded')
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function countRawCssRules(source: string): { rules: number; selectorComponents: number } {
|
|
131
|
+
let rules = 0
|
|
132
|
+
let selectorComponents = 0
|
|
133
|
+
let preludeLength = 0
|
|
134
|
+
let preludeStart = 0
|
|
135
|
+
let quote: '"' | "'" | null = null
|
|
136
|
+
let cursor = 0
|
|
137
|
+
while (cursor < source.length) {
|
|
138
|
+
const char = source[cursor]!
|
|
139
|
+
if (char === '\\') {
|
|
140
|
+
if (cursor + 1 >= source.length) throw new Error('Embedded stylesheet complexity limit exceeded')
|
|
141
|
+
cursor += 2
|
|
142
|
+
preludeLength += 2
|
|
143
|
+
continue
|
|
144
|
+
}
|
|
145
|
+
if (quote !== null) {
|
|
146
|
+
if (char === quote) quote = null
|
|
147
|
+
cursor++
|
|
148
|
+
continue
|
|
149
|
+
}
|
|
150
|
+
if (source.startsWith('/*', cursor)) {
|
|
151
|
+
const end = source.indexOf('*/', cursor + 2)
|
|
152
|
+
if (end < 0) throw new Error('Embedded stylesheet complexity limit exceeded')
|
|
153
|
+
cursor = end + 2
|
|
154
|
+
continue
|
|
155
|
+
}
|
|
156
|
+
if (char === '"' || char === "'") {
|
|
157
|
+
quote = char
|
|
158
|
+
cursor++
|
|
159
|
+
continue
|
|
160
|
+
}
|
|
161
|
+
if (char === '{') {
|
|
162
|
+
rules++
|
|
163
|
+
if (preludeLength > MAX_RAW_CSS_SEGMENT_CHARS || rules > MAX_EMBEDDED_STYLE_RULES) {
|
|
164
|
+
throw new Error('Embedded stylesheet complexity limit exceeded')
|
|
165
|
+
}
|
|
166
|
+
selectorComponents += countSelectorListComponents(source.slice(preludeStart, cursor), MAX_RAW_CSS_SEGMENT_CHARS)
|
|
167
|
+
preludeLength = 0
|
|
168
|
+
preludeStart = cursor + 1
|
|
169
|
+
} else if (char === '}' || char === ';') {
|
|
170
|
+
preludeLength = 0
|
|
171
|
+
preludeStart = cursor + 1
|
|
172
|
+
} else {
|
|
173
|
+
preludeLength++
|
|
174
|
+
if (preludeLength > MAX_RAW_CSS_SEGMENT_CHARS) throw new Error('Embedded stylesheet complexity limit exceeded')
|
|
175
|
+
}
|
|
176
|
+
cursor++
|
|
177
|
+
}
|
|
178
|
+
if (quote !== null) throw new Error('Embedded stylesheet complexity limit exceeded')
|
|
179
|
+
countSelectorListComponents(source.slice(preludeStart), MAX_RAW_CSS_SEGMENT_CHARS)
|
|
180
|
+
return { rules, selectorComponents }
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function buildComputedStyleResolver(
|
|
184
|
+
document: Document,
|
|
185
|
+
resolve: (element: Element) => CSSStyleDeclaration,
|
|
186
|
+
): ((element: Element) => CSSStyleDeclaration) | undefined {
|
|
187
|
+
const styles = Array.from(document.querySelectorAll('style'))
|
|
188
|
+
if (styles.length === 0) return undefined
|
|
189
|
+
|
|
190
|
+
let styleCharacters = 0
|
|
191
|
+
let ruleCount = 0
|
|
192
|
+
let selectorComponentCount = 0
|
|
193
|
+
for (const style of styles) {
|
|
194
|
+
styleCharacters += style.textContent?.length ?? 0
|
|
195
|
+
if (styleCharacters > MAX_EMBEDDED_STYLE_CHARS) throw new Error('Embedded stylesheet complexity limit exceeded')
|
|
196
|
+
const counted =
|
|
197
|
+
style.sheet === null
|
|
198
|
+
? { rules: MAX_EMBEDDED_STYLE_RULES + 1, selectorComponents: MAX_EMBEDDED_STYLE_RULES + 1 }
|
|
199
|
+
: countCssRules(style.sheet.cssRules)
|
|
200
|
+
ruleCount += counted.rules
|
|
201
|
+
selectorComponentCount += counted.selectorComponents
|
|
202
|
+
if (ruleCount > MAX_EMBEDDED_STYLE_RULES) throw new Error('Embedded stylesheet complexity limit exceeded')
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
const elementCount = document.querySelectorAll('*').length
|
|
206
|
+
if (selectorComponentCount * elementCount > MAX_COMPUTED_STYLE_WORK) {
|
|
207
|
+
throw new Error('Embedded stylesheet complexity limit exceeded')
|
|
208
|
+
}
|
|
209
|
+
return resolve
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function countCssRules(rules: CSSRuleList): { rules: number; selectorComponents: number } {
|
|
213
|
+
let count = 0
|
|
214
|
+
let selectorComponents = 0
|
|
215
|
+
const pending = Array.from(rules)
|
|
216
|
+
while (pending.length > 0) {
|
|
217
|
+
const rule = pending.pop()!
|
|
218
|
+
count++
|
|
219
|
+
if (count > MAX_EMBEDDED_STYLE_RULES) return { rules: count, selectorComponents }
|
|
220
|
+
const prelude =
|
|
221
|
+
'selectorText' in rule ? (rule as CSSStyleRule).selectorText : rule.cssText.slice(0, rule.cssText.indexOf('{'))
|
|
222
|
+
selectorComponents += countSelectorListComponents(prelude, MAX_SELECTOR_CHARS)
|
|
223
|
+
if ('cssRules' in rule) pending.push(...Array.from((rule as CSSGroupingRule).cssRules))
|
|
224
|
+
}
|
|
225
|
+
return { rules: count, selectorComponents }
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
function countSelectorListComponents(source: string, maxLength: number): number {
|
|
229
|
+
if (source.length > maxLength) throw new Error('Embedded stylesheet complexity limit exceeded')
|
|
230
|
+
let components = 1
|
|
231
|
+
let quote: '"' | "'" | null = null
|
|
232
|
+
let parentheses = 0
|
|
233
|
+
let brackets = 0
|
|
234
|
+
let cursor = 0
|
|
235
|
+
while (cursor < source.length) {
|
|
236
|
+
const char = source[cursor]!
|
|
237
|
+
if (char === '\\') {
|
|
238
|
+
if (cursor + 1 >= source.length) throw new Error('Embedded stylesheet complexity limit exceeded')
|
|
239
|
+
cursor += 2
|
|
240
|
+
continue
|
|
241
|
+
}
|
|
242
|
+
if (quote !== null) {
|
|
243
|
+
if (char === quote) quote = null
|
|
244
|
+
cursor++
|
|
245
|
+
continue
|
|
246
|
+
}
|
|
247
|
+
if (source.startsWith('/*', cursor)) {
|
|
248
|
+
const end = source.indexOf('*/', cursor + 2)
|
|
249
|
+
if (end < 0) throw new Error('Embedded stylesheet complexity limit exceeded')
|
|
250
|
+
cursor = end + 2
|
|
251
|
+
continue
|
|
252
|
+
}
|
|
253
|
+
if (char === '"' || char === "'") quote = char
|
|
254
|
+
else if (char === '(') parentheses++
|
|
255
|
+
else if (char === ')') {
|
|
256
|
+
if (parentheses === 0) throw new Error('Embedded stylesheet complexity limit exceeded')
|
|
257
|
+
parentheses--
|
|
258
|
+
} else if (char === '[') brackets++
|
|
259
|
+
else if (char === ']') {
|
|
260
|
+
if (brackets === 0) throw new Error('Embedded stylesheet complexity limit exceeded')
|
|
261
|
+
brackets--
|
|
262
|
+
} else if (char === ',' && parentheses === 0 && brackets === 0) components++
|
|
263
|
+
cursor++
|
|
264
|
+
}
|
|
265
|
+
if (quote !== null || parentheses !== 0 || brackets !== 0) {
|
|
266
|
+
throw new Error('Embedded stylesheet complexity limit exceeded')
|
|
267
|
+
}
|
|
268
|
+
return components
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
function countVisibleTextCharacters(body: HTMLElement | null): number {
|
|
272
|
+
if (body === null) return 0
|
|
273
|
+
return Array.from(normalizeVisibleText(body.textContent ?? '')).length
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
function pruneHiddenContent(
|
|
277
|
+
document: Document,
|
|
278
|
+
computedStyle: ((element: Element) => CSSStyleDeclaration) | undefined,
|
|
279
|
+
): void {
|
|
280
|
+
for (const details of document.querySelectorAll('details:not([open])')) pruneClosedDetails(details)
|
|
281
|
+
|
|
282
|
+
for (const element of document.querySelectorAll('*')) {
|
|
283
|
+
if (isHiddenElement(element, computedStyle)) element.remove()
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
function pruneClosedDetails(details: Element): void {
|
|
288
|
+
const summary = Array.from(details.children).find((child) => child.tagName === 'SUMMARY')
|
|
289
|
+
for (const child of Array.from(details.childNodes)) {
|
|
290
|
+
if (child !== summary) child.remove()
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
function isHiddenElement(
|
|
295
|
+
element: Element,
|
|
296
|
+
computedStyle: ((element: Element) => CSSStyleDeclaration) | undefined,
|
|
297
|
+
): boolean {
|
|
298
|
+
if (element.hasAttribute('hidden')) return true
|
|
299
|
+
if (element.getAttribute('aria-hidden')?.toLocaleLowerCase() === 'true') return true
|
|
300
|
+
if (element.tagName === 'INPUT' && element.getAttribute('type')?.toLocaleLowerCase() === 'hidden') return true
|
|
301
|
+
if (element.tagName === 'DIALOG' && !element.hasAttribute('open')) return true
|
|
302
|
+
|
|
303
|
+
const rawStyle = element.getAttribute('style')
|
|
304
|
+
if (rawStyle !== null && classifyRawInlineStyleVisibility(rawStyle) !== 'visible') return true
|
|
305
|
+
|
|
306
|
+
if ('style' in element) {
|
|
307
|
+
const { position, left, top } = (element as HTMLElement).style
|
|
308
|
+
if ((position === 'absolute' || position === 'fixed') && (isFarOffscreen(left) || isFarOffscreen(top))) return true
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
if (computedStyle === undefined || NON_CONTENT_ELEMENTS.has(element.tagName)) return false
|
|
312
|
+
let resolved: CSSStyleDeclaration
|
|
313
|
+
try {
|
|
314
|
+
resolved = computedStyle(element)
|
|
315
|
+
} catch {
|
|
316
|
+
return false
|
|
317
|
+
}
|
|
318
|
+
if (resolved.display === 'none' || resolved.visibility === 'hidden' || resolved.visibility === 'collapse') return true
|
|
319
|
+
if (resolved.getPropertyValue('content-visibility').trim().toLocaleLowerCase() === 'hidden') return true
|
|
320
|
+
const opacity = Number(resolved.opacity)
|
|
321
|
+
if (resolved.opacity !== '' && Number.isFinite(opacity) && opacity <= 0) return true
|
|
322
|
+
|
|
323
|
+
return false
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
const NON_CONTENT_ELEMENTS = new Set([
|
|
327
|
+
'HEAD',
|
|
328
|
+
'LINK',
|
|
329
|
+
'META',
|
|
330
|
+
'NOFRAMES',
|
|
331
|
+
'NOSCRIPT',
|
|
332
|
+
'SCRIPT',
|
|
333
|
+
'STYLE',
|
|
334
|
+
'TEMPLATE',
|
|
335
|
+
'TITLE',
|
|
336
|
+
])
|
|
337
|
+
|
|
338
|
+
function isFarOffscreen(value: string): boolean {
|
|
339
|
+
const match = /^(-?(?:\d+\.?\d*|\.\d+))(px|rem|em|vw|vh)$/iu.exec(value.trim())
|
|
340
|
+
return match !== null && Number(match[1]) <= -1_000
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
function normalizeVisibleText(value: string): string {
|
|
344
|
+
return value.replace(/\s+/gu, ' ').trim()
|
|
345
|
+
}
|
|
46
346
|
|
|
347
|
+
function renderOutput(markdown: string, title: string | null | undefined): string {
|
|
47
348
|
if (!markdown) return 'Readability extracted no content from this page.'
|
|
48
349
|
|
|
49
|
-
if (
|
|
50
|
-
return `# ${
|
|
350
|
+
if (title) {
|
|
351
|
+
return `# ${title}\n\n${markdown}`
|
|
51
352
|
}
|
|
52
353
|
return markdown
|
|
53
354
|
}
|