crawlutron-extract 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- crawlutron_extract/__init__.py +36 -0
- crawlutron_extract/_extract.py +338 -0
- crawlutron_extract/_html_common.py +49 -0
- crawlutron_extract/_select_common.py +84 -0
- crawlutron_extract/charset.py +94 -0
- crawlutron_extract/css_select.py +447 -0
- crawlutron_extract/html_diagnostics.py +72 -0
- crawlutron_extract/html_dom.py +114 -0
- crawlutron_extract/html_forms_frames.py +109 -0
- crawlutron_extract/html_links.py +84 -0
- crawlutron_extract/html_meta.py +160 -0
- crawlutron_extract/html_pass.py +212 -0
- crawlutron_extract/html_tables.py +213 -0
- crawlutron_extract/html_text.py +211 -0
- crawlutron_extract/js_endpoints.py +55 -0
- crawlutron_extract/limits.py +59 -0
- crawlutron_extract/links.py +109 -0
- crawlutron_extract/models.py +39 -0
- crawlutron_extract/office.py +360 -0
- crawlutron_extract/pagination.py +50 -0
- crawlutron_extract/pdf_pass.py +351 -0
- crawlutron_extract/py.typed +0 -0
- crawlutron_extract/schema.py +248 -0
- crawlutron_extract/sitemap.py +59 -0
- crawlutron_extract/text_format.py +281 -0
- crawlutron_extract/xpath_eval.py +143 -0
- crawlutron_extract/xpath_select.py +283 -0
- crawlutron_extract-0.5.0.dist-info/METADATA +233 -0
- crawlutron_extract-0.5.0.dist-info/RECORD +30 -0
- crawlutron_extract-0.5.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""crawlutron_extract: stdlib-only content extraction, bounded and deterministic."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from crawlutron_extract._extract import DEFAULT_LIMITS, extract
|
|
6
|
+
from crawlutron_extract.css_select import SelectorError, validate_css
|
|
7
|
+
from crawlutron_extract.html_diagnostics import html_diagnostics
|
|
8
|
+
from crawlutron_extract.js_endpoints import extract_js_endpoints
|
|
9
|
+
from crawlutron_extract.limits import ExtractLimits
|
|
10
|
+
from crawlutron_extract.links import LinkRecord, extract_discovered_links
|
|
11
|
+
from crawlutron_extract.models import Extraction, Quality
|
|
12
|
+
from crawlutron_extract.pagination import detect_pagination
|
|
13
|
+
from crawlutron_extract.schema import FieldKind, FieldSpec, Schema, extract_records
|
|
14
|
+
from crawlutron_extract.sitemap import extract_sitemap_locs
|
|
15
|
+
from crawlutron_extract.xpath_select import validate_xpath
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"DEFAULT_LIMITS",
|
|
19
|
+
"ExtractLimits",
|
|
20
|
+
"Extraction",
|
|
21
|
+
"FieldKind",
|
|
22
|
+
"FieldSpec",
|
|
23
|
+
"Quality",
|
|
24
|
+
"Schema",
|
|
25
|
+
"extract",
|
|
26
|
+
"extract_discovered_links",
|
|
27
|
+
"extract_js_endpoints",
|
|
28
|
+
"extract_records",
|
|
29
|
+
"extract_sitemap_locs",
|
|
30
|
+
"html_diagnostics",
|
|
31
|
+
"LinkRecord",
|
|
32
|
+
"SelectorError",
|
|
33
|
+
"detect_pagination",
|
|
34
|
+
"validate_css",
|
|
35
|
+
"validate_xpath",
|
|
36
|
+
]
|
|
@@ -0,0 +1,338 @@
|
|
|
1
|
+
"""The extract() entry point: bounded, deterministic, never-raise.
|
|
2
|
+
|
|
3
|
+
Slice 3: HTML single-pass extraction with quality ladder.
|
|
4
|
+
Slice 4: CSV/JSON/XML text format extraction.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import cast
|
|
10
|
+
|
|
11
|
+
from safe_url import HostPolicy, NormalizedUrl
|
|
12
|
+
|
|
13
|
+
from crawlutron_extract.charset import decode_content
|
|
14
|
+
from crawlutron_extract.html_pass import extract_html
|
|
15
|
+
from crawlutron_extract.limits import RICH_TEXT_FLOOR, ExtractLimits
|
|
16
|
+
from crawlutron_extract.models import Extraction, Quality
|
|
17
|
+
from crawlutron_extract.office import extract_office
|
|
18
|
+
from crawlutron_extract.pdf_pass import extract_pdf
|
|
19
|
+
from crawlutron_extract.text_format import extract_text_format
|
|
20
|
+
|
|
21
|
+
__all__ = ["DEFAULT_LIMITS", "extract"]
|
|
22
|
+
|
|
23
|
+
DEFAULT_LIMITS = ExtractLimits()
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _resolve_quality(
|
|
27
|
+
*,
|
|
28
|
+
content_type: str,
|
|
29
|
+
text: str,
|
|
30
|
+
flags: list[str],
|
|
31
|
+
found_main: bool,
|
|
32
|
+
html_links: list,
|
|
33
|
+
html_frames: list,
|
|
34
|
+
html_forms: list,
|
|
35
|
+
html_tables: list,
|
|
36
|
+
) -> Quality:
|
|
37
|
+
"""Apply the quality ladder: FAILED < LOW < MEDIUM < HIGH."""
|
|
38
|
+
lossy = "charset_lossy" in flags
|
|
39
|
+
has_body_text = len(text.strip()) > 0
|
|
40
|
+
has_content = (
|
|
41
|
+
has_body_text
|
|
42
|
+
or len(html_links) > 0
|
|
43
|
+
or len(html_frames) > 0
|
|
44
|
+
or len(html_forms) > 0
|
|
45
|
+
or len(html_tables) > 0
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
if not has_content:
|
|
49
|
+
return Quality.FAILED
|
|
50
|
+
|
|
51
|
+
if "text/html" not in content_type.lower():
|
|
52
|
+
# Non-HTML: no main-content concept; decoded text alone is LOW.
|
|
53
|
+
return Quality.LOW
|
|
54
|
+
|
|
55
|
+
if lossy:
|
|
56
|
+
return Quality.LOW
|
|
57
|
+
|
|
58
|
+
if not has_body_text:
|
|
59
|
+
# Only boilerplate/links/frames/forms were found.
|
|
60
|
+
return Quality.LOW
|
|
61
|
+
|
|
62
|
+
if found_main and len(text) >= RICH_TEXT_FLOOR:
|
|
63
|
+
return Quality.HIGH
|
|
64
|
+
|
|
65
|
+
return Quality.MEDIUM
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _resolve_text_format_quality(quality_str: str) -> Quality:
|
|
69
|
+
"""Map text format quality string to Quality enum."""
|
|
70
|
+
mapping = {
|
|
71
|
+
"high": Quality.HIGH,
|
|
72
|
+
"medium": Quality.MEDIUM,
|
|
73
|
+
"low": Quality.LOW,
|
|
74
|
+
"failed": Quality.FAILED,
|
|
75
|
+
}
|
|
76
|
+
return mapping.get(quality_str, Quality.FAILED)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _apply_text_format(
|
|
80
|
+
fmt: dict[str, object],
|
|
81
|
+
metadata: dict[str, str],
|
|
82
|
+
flags: list[str],
|
|
83
|
+
) -> tuple[
|
|
84
|
+
str,
|
|
85
|
+
str,
|
|
86
|
+
list[NormalizedUrl],
|
|
87
|
+
list,
|
|
88
|
+
list,
|
|
89
|
+
list[list[list[str]]],
|
|
90
|
+
bool,
|
|
91
|
+
Quality,
|
|
92
|
+
]:
|
|
93
|
+
"""Apply a text-format result dict to local variables.
|
|
94
|
+
|
|
95
|
+
Shared by CSV/JSON/XML branches to avoid duplicating the same
|
|
96
|
+
dict-unpacking logic three times.
|
|
97
|
+
"""
|
|
98
|
+
metadata.update(cast(dict[str, str], fmt["metadata"]))
|
|
99
|
+
text = cast(str, fmt["text"])
|
|
100
|
+
markdown = cast(str, fmt["markdown"])
|
|
101
|
+
links = cast(list[NormalizedUrl], fmt["links"])
|
|
102
|
+
frames = cast(list[dict[str, str]], fmt["frames"])
|
|
103
|
+
forms = cast(list[dict[str, str]], fmt["forms"])
|
|
104
|
+
tables = cast(list[list[list[str]]], fmt["tables"])
|
|
105
|
+
truncated = cast(bool, fmt["truncated"])
|
|
106
|
+
flags.extend(cast(list[str], fmt.get("quality_flags", [])))
|
|
107
|
+
quality = _resolve_text_format_quality(cast(str, fmt.get("_quality", "failed")))
|
|
108
|
+
return text, markdown, links, frames, forms, tables, truncated, quality
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _empty_extraction(
|
|
112
|
+
*,
|
|
113
|
+
quality: Quality,
|
|
114
|
+
flag: str,
|
|
115
|
+
flags: list[str],
|
|
116
|
+
metadata: dict[str, str],
|
|
117
|
+
charset: str | None,
|
|
118
|
+
) -> Extraction:
|
|
119
|
+
"""Build an empty Extraction with the given quality and flag."""
|
|
120
|
+
return Extraction(
|
|
121
|
+
quality=quality,
|
|
122
|
+
flags=flags + [flag],
|
|
123
|
+
metadata=metadata,
|
|
124
|
+
text="",
|
|
125
|
+
markdown="",
|
|
126
|
+
links=[],
|
|
127
|
+
frames=[],
|
|
128
|
+
forms=[],
|
|
129
|
+
tables=[],
|
|
130
|
+
truncated=False,
|
|
131
|
+
charset=charset,
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def extract(
|
|
136
|
+
content: bytes,
|
|
137
|
+
content_type: str,
|
|
138
|
+
*,
|
|
139
|
+
url: str | None = None,
|
|
140
|
+
limits: ExtractLimits = DEFAULT_LIMITS,
|
|
141
|
+
) -> Extraction:
|
|
142
|
+
"""Extract bounded, deterministic output. Refuses oversized input.
|
|
143
|
+
|
|
144
|
+
Never raises on hostile input; returns quality="failed" instead."""
|
|
145
|
+
if len(content) > limits.max_input_bytes:
|
|
146
|
+
return Extraction(quality=Quality.FAILED, flags=["input_refused"])
|
|
147
|
+
metadata: dict[str, str] = {}
|
|
148
|
+
if content_type:
|
|
149
|
+
metadata["content_type"] = content_type
|
|
150
|
+
if url is not None:
|
|
151
|
+
metadata["url"] = url
|
|
152
|
+
|
|
153
|
+
ctype = content_type.lower()
|
|
154
|
+
container_types = (
|
|
155
|
+
"application/msword", # .doc legacy binary — refused with flag
|
|
156
|
+
)
|
|
157
|
+
is_office = (
|
|
158
|
+
ctype == "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
|
|
159
|
+
or ctype == "application/vnd.openxmlformats-officedocument.presentationml.presentation"
|
|
160
|
+
or ctype == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
|
161
|
+
or ctype == "application/epub+zip"
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
# Binary containers (PDF, DOC) skip the charset ladder — decode_content
|
|
165
|
+
# would emit charset_lossy on non-text bytes.
|
|
166
|
+
flags: list[str] = []
|
|
167
|
+
if ctype == "application/pdf":
|
|
168
|
+
# PDF tier-1 text sweep; below the quality floor returns empty + low.
|
|
169
|
+
fmt = extract_pdf(content, limits)
|
|
170
|
+
(
|
|
171
|
+
pdf_text,
|
|
172
|
+
pdf_markdown,
|
|
173
|
+
pdf_links,
|
|
174
|
+
pdf_frames,
|
|
175
|
+
pdf_forms,
|
|
176
|
+
pdf_tables,
|
|
177
|
+
pdf_truncated,
|
|
178
|
+
pdf_quality,
|
|
179
|
+
) = _apply_text_format(fmt, metadata, flags)
|
|
180
|
+
return Extraction(
|
|
181
|
+
quality=pdf_quality,
|
|
182
|
+
flags=flags,
|
|
183
|
+
metadata=metadata,
|
|
184
|
+
text=pdf_text,
|
|
185
|
+
markdown=pdf_markdown,
|
|
186
|
+
links=pdf_links,
|
|
187
|
+
frames=pdf_frames,
|
|
188
|
+
forms=pdf_forms,
|
|
189
|
+
tables=pdf_tables,
|
|
190
|
+
truncated=pdf_truncated,
|
|
191
|
+
charset=None,
|
|
192
|
+
links_refused=0,
|
|
193
|
+
)
|
|
194
|
+
if ctype in container_types:
|
|
195
|
+
charset: str | None = None
|
|
196
|
+
return _empty_extraction(
|
|
197
|
+
quality=Quality.LOW,
|
|
198
|
+
flag=f"{ctype.split('/')[-1]}_unsupported",
|
|
199
|
+
flags=flags,
|
|
200
|
+
metadata=metadata,
|
|
201
|
+
charset=charset,
|
|
202
|
+
)
|
|
203
|
+
# Office containers (DOCX/PPTX/XLSX/EPUB) extracted via zip+XML.
|
|
204
|
+
if is_office:
|
|
205
|
+
fmt = extract_office(content, content_type, limits)
|
|
206
|
+
(
|
|
207
|
+
of_text,
|
|
208
|
+
of_markdown,
|
|
209
|
+
of_links,
|
|
210
|
+
of_frames,
|
|
211
|
+
of_forms,
|
|
212
|
+
of_tables,
|
|
213
|
+
of_truncated,
|
|
214
|
+
of_quality,
|
|
215
|
+
) = _apply_text_format(fmt, metadata, flags)
|
|
216
|
+
return Extraction(
|
|
217
|
+
quality=of_quality,
|
|
218
|
+
flags=flags,
|
|
219
|
+
metadata=metadata,
|
|
220
|
+
text=of_text,
|
|
221
|
+
markdown=of_markdown,
|
|
222
|
+
links=of_links,
|
|
223
|
+
frames=of_frames,
|
|
224
|
+
forms=of_forms,
|
|
225
|
+
tables=of_tables,
|
|
226
|
+
truncated=of_truncated,
|
|
227
|
+
charset=None,
|
|
228
|
+
links_refused=0,
|
|
229
|
+
)
|
|
230
|
+
# Truly unsupported type: refuse without decode.
|
|
231
|
+
if not (
|
|
232
|
+
ctype == "text/html"
|
|
233
|
+
or ctype.startswith("text/html;")
|
|
234
|
+
or ctype == "text/csv"
|
|
235
|
+
or ctype.startswith("text/csv;")
|
|
236
|
+
or ctype == "application/csv"
|
|
237
|
+
or ctype == "application/json"
|
|
238
|
+
or ctype.startswith("application/json;")
|
|
239
|
+
or ctype == "application/jsonl"
|
|
240
|
+
or ctype.startswith("application/jsonl;")
|
|
241
|
+
or ctype == "text/json"
|
|
242
|
+
or ctype == "application/xml"
|
|
243
|
+
or ctype.startswith("application/xml;")
|
|
244
|
+
or ctype == "text/xml"
|
|
245
|
+
or ctype.startswith("text/xml;")
|
|
246
|
+
or ctype.startswith("text/")
|
|
247
|
+
):
|
|
248
|
+
return _empty_extraction(
|
|
249
|
+
quality=Quality.FAILED,
|
|
250
|
+
flag="format_refused",
|
|
251
|
+
flags=[],
|
|
252
|
+
metadata=metadata,
|
|
253
|
+
charset=None,
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
text, charset, flags = decode_content(content, content_type)
|
|
257
|
+
if charset:
|
|
258
|
+
metadata["charset"] = charset
|
|
259
|
+
|
|
260
|
+
markdown = ""
|
|
261
|
+
links: list = []
|
|
262
|
+
frames: list = []
|
|
263
|
+
forms: list = []
|
|
264
|
+
tables: list[list[list[str]]] = []
|
|
265
|
+
truncated = False
|
|
266
|
+
quality = Quality.FAILED
|
|
267
|
+
links_refused = 0
|
|
268
|
+
|
|
269
|
+
is_html = ctype == "text/html" or ctype.startswith("text/html;")
|
|
270
|
+
is_csv = ctype == "text/csv" or ctype.startswith("text/csv;") or ctype == "application/csv"
|
|
271
|
+
is_json = (
|
|
272
|
+
ctype == "application/json"
|
|
273
|
+
or ctype.startswith("application/json;")
|
|
274
|
+
or ctype == "application/jsonl"
|
|
275
|
+
or ctype.startswith("application/jsonl;")
|
|
276
|
+
or ctype == "text/json"
|
|
277
|
+
)
|
|
278
|
+
is_xml = (
|
|
279
|
+
ctype == "application/xml"
|
|
280
|
+
or ctype.startswith("application/xml;")
|
|
281
|
+
or ctype == "text/xml"
|
|
282
|
+
or ctype.startswith("text/xml;")
|
|
283
|
+
)
|
|
284
|
+
if is_html:
|
|
285
|
+
policy = HostPolicy.unrestricted()
|
|
286
|
+
html = extract_html(text, limits, base_url=url, policy=policy)
|
|
287
|
+
metadata.update(html["metadata"])
|
|
288
|
+
text = html["text"]
|
|
289
|
+
markdown = html["markdown"]
|
|
290
|
+
links = html["links"]
|
|
291
|
+
frames = html["frames"]
|
|
292
|
+
forms = html["forms"]
|
|
293
|
+
tables = html["tables"]
|
|
294
|
+
truncated = html["truncated"]
|
|
295
|
+
if html["links_refused"] > 0:
|
|
296
|
+
flags.append("links_refused")
|
|
297
|
+
flags.extend(html.get("flags", []))
|
|
298
|
+
links_refused = html["links_refused"]
|
|
299
|
+
quality = _resolve_quality(
|
|
300
|
+
content_type=content_type,
|
|
301
|
+
text=text,
|
|
302
|
+
flags=flags,
|
|
303
|
+
found_main=html["found_main"],
|
|
304
|
+
html_links=links,
|
|
305
|
+
html_frames=frames,
|
|
306
|
+
html_forms=forms,
|
|
307
|
+
html_tables=tables,
|
|
308
|
+
)
|
|
309
|
+
elif is_csv or is_json or is_xml:
|
|
310
|
+
fmt = extract_text_format(text, content_type, limits)
|
|
311
|
+
(
|
|
312
|
+
text,
|
|
313
|
+
markdown,
|
|
314
|
+
links,
|
|
315
|
+
frames,
|
|
316
|
+
forms,
|
|
317
|
+
tables,
|
|
318
|
+
truncated,
|
|
319
|
+
quality,
|
|
320
|
+
) = _apply_text_format(fmt, metadata, flags)
|
|
321
|
+
elif ctype.startswith("text/"):
|
|
322
|
+
# Plain text: no format extractor ran. Empty → FAILED, else LOW.
|
|
323
|
+
quality = Quality.LOW if text.strip() else Quality.FAILED
|
|
324
|
+
|
|
325
|
+
return Extraction(
|
|
326
|
+
quality=quality,
|
|
327
|
+
flags=flags,
|
|
328
|
+
metadata=metadata,
|
|
329
|
+
text=text,
|
|
330
|
+
markdown=markdown,
|
|
331
|
+
links=links,
|
|
332
|
+
frames=frames,
|
|
333
|
+
forms=forms,
|
|
334
|
+
tables=tables,
|
|
335
|
+
truncated=truncated,
|
|
336
|
+
charset=charset,
|
|
337
|
+
links_refused=links_refused,
|
|
338
|
+
)
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Shared HTML parse helpers over html.parser attr lists."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from html import parser as html_parser
|
|
6
|
+
|
|
7
|
+
from safe_url import HostPolicy, NormalizedUrl, admit_or_none
|
|
8
|
+
|
|
9
|
+
from crawlutron_extract.limits import ExtractLimits
|
|
10
|
+
|
|
11
|
+
__all__ = ["resolve_or_admit", "tag_attrs"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class _HtmlParser(html_parser.HTMLParser):
|
|
15
|
+
"""Base parser for single-pass HTML discovery: charrefs converted."""
|
|
16
|
+
|
|
17
|
+
def __init__(self) -> None:
|
|
18
|
+
super().__init__(convert_charrefs=True)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def resolve_or_admit(href: str, target: NormalizedUrl | None) -> NormalizedUrl | None:
|
|
22
|
+
"""Resolve href against target, or admit it outright when there is no base URL."""
|
|
23
|
+
if target is not None:
|
|
24
|
+
return target.resolve(href)
|
|
25
|
+
return admit_or_none(href, policy=HostPolicy.unrestricted())
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _prepare(
|
|
29
|
+
html: bytes | str,
|
|
30
|
+
source_url: str | NormalizedUrl | None,
|
|
31
|
+
limits: ExtractLimits | None = None,
|
|
32
|
+
) -> tuple[str, NormalizedUrl | None, ExtractLimits]:
|
|
33
|
+
"""Decode html, admit the source URL, and apply the default limits."""
|
|
34
|
+
if limits is None:
|
|
35
|
+
limits = ExtractLimits()
|
|
36
|
+
text = html.decode("utf-8", "replace") if isinstance(html, bytes) else html
|
|
37
|
+
base: NormalizedUrl | None = None
|
|
38
|
+
if isinstance(source_url, NormalizedUrl):
|
|
39
|
+
base = source_url
|
|
40
|
+
elif source_url is not None:
|
|
41
|
+
admitted = admit_or_none(source_url, policy=HostPolicy.unrestricted())
|
|
42
|
+
if admitted is not None:
|
|
43
|
+
base = admitted
|
|
44
|
+
return text, base, limits
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def tag_attrs(attrs: list[tuple[str, str | None]]) -> dict[str, str]:
|
|
48
|
+
"""html.parser's (name, value) pairs as a string dict, values normalised."""
|
|
49
|
+
return {name: (value or "") for name, value in attrs}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Shared tokenizer helpers and the shared ``SelectorError`` for the CSS and
|
|
2
|
+
XPath selector parsers, kept in one module to avoid jscpd clones.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from typing import Never
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class SelectorError(Exception):
|
|
11
|
+
"""Raised when a selector uses syntax outside the supported subset."""
|
|
12
|
+
|
|
13
|
+
def __init__(self, message: str, selector: str = "", token: str | None = None):
|
|
14
|
+
sel = f" (selector: {selector!r})" if selector else ""
|
|
15
|
+
tail = f" (token: {token!r})" if token is not None else ""
|
|
16
|
+
super().__init__(f"{message}{sel}{tail}")
|
|
17
|
+
self.selector = selector
|
|
18
|
+
self.expression = selector
|
|
19
|
+
self.token = token
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _peek(tokens: list[tuple[str, str]], pos: int) -> tuple[str, str] | None:
|
|
23
|
+
if 0 <= pos < len(tokens):
|
|
24
|
+
return tokens[pos]
|
|
25
|
+
return None
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _consume(
|
|
29
|
+
tokens: list[tuple[str, str]], pos: int, expected: str | None = None
|
|
30
|
+
) -> tuple[tuple[str, str], int]:
|
|
31
|
+
if pos >= len(tokens):
|
|
32
|
+
raise SelectorError("Unexpected end of expression")
|
|
33
|
+
tok = tokens[pos]
|
|
34
|
+
if expected and tok[0] != expected:
|
|
35
|
+
raise SelectorError(f"Expected {expected}, got {tok[0]}", token=tok[1])
|
|
36
|
+
return tok, pos + 1
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _skip_ws(tokens: list[tuple[str, str]], pos: int) -> int:
|
|
40
|
+
tok = _peek(tokens, pos)
|
|
41
|
+
while tok and tok[0] == "WS":
|
|
42
|
+
_, pos = _consume(tokens, pos, "WS")
|
|
43
|
+
tok = _peek(tokens, pos)
|
|
44
|
+
return pos
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _emit_char(tokens: list[tuple[str, str]], kind: str, char: str) -> None:
|
|
48
|
+
tokens.append((kind, char))
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _scan_ident(text: str, i: int, n: int, extra: str = "-_.") -> int:
|
|
52
|
+
j = i
|
|
53
|
+
while j < n and (text[j].isalnum() or text[j] in extra):
|
|
54
|
+
j += 1
|
|
55
|
+
return j
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
FUNC_NAMES = frozenset({"contains", "starts-with", "normalize-space", "concat"})
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _is_func_call(tok: tuple[str, str] | None) -> bool:
|
|
62
|
+
return tok is not None and tok[0] == "NAME" and tok[1] in FUNC_NAMES
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _consume_parens(tokens: list[tuple[str, str]], pos: int) -> int:
|
|
66
|
+
_, pos = _consume(tokens, pos, "LPAREN")
|
|
67
|
+
_, pos = _consume(tokens, pos, "RPAREN")
|
|
68
|
+
return pos
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _consume_attr(tokens: list[tuple[str, str]], pos: int) -> int:
|
|
72
|
+
_, pos = _consume(tokens, pos, "AT")
|
|
73
|
+
return pos
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _parse_string_literal(
|
|
77
|
+
tok: tuple[str, str], tokens: list[tuple[str, str]], pos: int
|
|
78
|
+
) -> tuple[tuple[str, ...], int]:
|
|
79
|
+
_, pos = _consume(tokens, pos, "STRING")
|
|
80
|
+
return ("STRING_LITERAL", tok[1]), pos
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _unexpected_end(tokens: list[tuple[str, str]], pos: int, message: str, expression: str) -> Never:
|
|
84
|
+
raise SelectorError(message, expression, None)
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""Charset detection and lossy-safe decoding.
|
|
2
|
+
|
|
3
|
+
BOM -> Content-Type -> <meta charset> -> strict UTF-8 -> lossy. Never raises.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import re
|
|
9
|
+
|
|
10
|
+
__all__ = ["decode_content"]
|
|
11
|
+
|
|
12
|
+
# BOM -> charset, longest BOM first so UTF-8 (3 bytes) wins over UTF-16 (2 bytes).
|
|
13
|
+
_BOM_CHARS = (
|
|
14
|
+
(b"\xef\xbb\xbf", "utf-8"),
|
|
15
|
+
(b"\xff\xfe", "utf-16-le"),
|
|
16
|
+
(b"\xfe\xff", "utf-16-be"),
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
# Bounded meta-charset scan: first 8 KiB only, no full HTML parse.
|
|
20
|
+
_META_CHARSET_RE = re.compile(
|
|
21
|
+
r'<meta[^>]+charset\s*=\s*["\']?([^\s"\']+)',
|
|
22
|
+
re.IGNORECASE,
|
|
23
|
+
)
|
|
24
|
+
_META_CONTENT_TYPE_RE = re.compile(
|
|
25
|
+
r'<meta[^>]+http-equiv\s*=\s*["\']?content-type["\']?'
|
|
26
|
+
r'[^>]+content\s*=\s*["\']?[^;\s"]*;\s*charset\s*=\s*([^\s"\']+)',
|
|
27
|
+
re.IGNORECASE,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
_META_SCAN_BYTES = 8192
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _strip_bom(content: bytes) -> tuple[bytes, str | None]:
|
|
34
|
+
"""Strip a leading BOM, returning (body, charset) or (content, None)."""
|
|
35
|
+
for bom, charset in _BOM_CHARS:
|
|
36
|
+
if content.startswith(bom):
|
|
37
|
+
return content[len(bom) :], charset
|
|
38
|
+
return content, None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _parse_content_type_charset(content_type: str) -> str | None:
|
|
42
|
+
"""Extract charset= from Content-Type, with or without quotes."""
|
|
43
|
+
if not content_type:
|
|
44
|
+
return None
|
|
45
|
+
match = re.search(r";\s*charset\s*=\s*[\"']?([^\s\";]+)", content_type, re.IGNORECASE)
|
|
46
|
+
return match.group(1).strip().lower() if match else None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _scan_meta_charset(content: bytes) -> str | None:
|
|
50
|
+
"""Scan the first 8 KiB for an HTML meta charset declaration."""
|
|
51
|
+
head = content[:_META_SCAN_BYTES]
|
|
52
|
+
text = head.decode("utf-8", errors="replace")
|
|
53
|
+
for pattern in (_META_CHARSET_RE, _META_CONTENT_TYPE_RE):
|
|
54
|
+
match = pattern.search(text)
|
|
55
|
+
if match:
|
|
56
|
+
return match.group(1).strip().lower()
|
|
57
|
+
return None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _detect_charset(content: bytes, content_type: str) -> str:
|
|
61
|
+
"""Resolve charset by priority: BOM -> Content-Type -> meta -> UTF-8."""
|
|
62
|
+
_, bom_charset = _strip_bom(content)
|
|
63
|
+
if bom_charset:
|
|
64
|
+
return bom_charset
|
|
65
|
+
|
|
66
|
+
declared = _parse_content_type_charset(content_type)
|
|
67
|
+
if declared:
|
|
68
|
+
return declared
|
|
69
|
+
|
|
70
|
+
if "text/html" in content_type.lower():
|
|
71
|
+
meta = _scan_meta_charset(content)
|
|
72
|
+
if meta:
|
|
73
|
+
return meta
|
|
74
|
+
|
|
75
|
+
return "utf-8"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def decode_content(content: bytes, content_type: str) -> tuple[str, str, list[str]]:
|
|
79
|
+
"""Decode content, never raising: returns (text, charset, flags).
|
|
80
|
+
|
|
81
|
+
Decode failure is reported only through the charset_lossy flag.
|
|
82
|
+
"""
|
|
83
|
+
flags: list[str] = []
|
|
84
|
+
body, _bom_charset = _strip_bom(content)
|
|
85
|
+
|
|
86
|
+
charset = _detect_charset(content, content_type)
|
|
87
|
+
|
|
88
|
+
try:
|
|
89
|
+
text = body.decode(charset)
|
|
90
|
+
except UnicodeDecodeError, LookupError:
|
|
91
|
+
flags.append("charset_lossy")
|
|
92
|
+
text = body.decode("utf-8", errors="replace")
|
|
93
|
+
|
|
94
|
+
return text, charset, flags
|