crawlutron-extract 0.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,36 @@
1
+ """crawlutron_extract: stdlib-only content extraction, bounded and deterministic."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from crawlutron_extract._extract import DEFAULT_LIMITS, extract
6
+ from crawlutron_extract.css_select import SelectorError, validate_css
7
+ from crawlutron_extract.html_diagnostics import html_diagnostics
8
+ from crawlutron_extract.js_endpoints import extract_js_endpoints
9
+ from crawlutron_extract.limits import ExtractLimits
10
+ from crawlutron_extract.links import LinkRecord, extract_discovered_links
11
+ from crawlutron_extract.models import Extraction, Quality
12
+ from crawlutron_extract.pagination import detect_pagination
13
+ from crawlutron_extract.schema import FieldKind, FieldSpec, Schema, extract_records
14
+ from crawlutron_extract.sitemap import extract_sitemap_locs
15
+ from crawlutron_extract.xpath_select import validate_xpath
16
+
17
+ __all__ = [
18
+ "DEFAULT_LIMITS",
19
+ "ExtractLimits",
20
+ "Extraction",
21
+ "FieldKind",
22
+ "FieldSpec",
23
+ "Quality",
24
+ "Schema",
25
+ "extract",
26
+ "extract_discovered_links",
27
+ "extract_js_endpoints",
28
+ "extract_records",
29
+ "extract_sitemap_locs",
30
+ "html_diagnostics",
31
+ "LinkRecord",
32
+ "SelectorError",
33
+ "detect_pagination",
34
+ "validate_css",
35
+ "validate_xpath",
36
+ ]
@@ -0,0 +1,338 @@
1
+ """The extract() entry point: bounded, deterministic, never-raise.
2
+
3
+ Slice 3: HTML single-pass extraction with quality ladder.
4
+ Slice 4: CSV/JSON/XML text format extraction.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from typing import cast
10
+
11
+ from safe_url import HostPolicy, NormalizedUrl
12
+
13
+ from crawlutron_extract.charset import decode_content
14
+ from crawlutron_extract.html_pass import extract_html
15
+ from crawlutron_extract.limits import RICH_TEXT_FLOOR, ExtractLimits
16
+ from crawlutron_extract.models import Extraction, Quality
17
+ from crawlutron_extract.office import extract_office
18
+ from crawlutron_extract.pdf_pass import extract_pdf
19
+ from crawlutron_extract.text_format import extract_text_format
20
+
21
+ __all__ = ["DEFAULT_LIMITS", "extract"]
22
+
23
+ DEFAULT_LIMITS = ExtractLimits()
24
+
25
+
26
+ def _resolve_quality(
27
+ *,
28
+ content_type: str,
29
+ text: str,
30
+ flags: list[str],
31
+ found_main: bool,
32
+ html_links: list,
33
+ html_frames: list,
34
+ html_forms: list,
35
+ html_tables: list,
36
+ ) -> Quality:
37
+ """Apply the quality ladder: FAILED < LOW < MEDIUM < HIGH."""
38
+ lossy = "charset_lossy" in flags
39
+ has_body_text = len(text.strip()) > 0
40
+ has_content = (
41
+ has_body_text
42
+ or len(html_links) > 0
43
+ or len(html_frames) > 0
44
+ or len(html_forms) > 0
45
+ or len(html_tables) > 0
46
+ )
47
+
48
+ if not has_content:
49
+ return Quality.FAILED
50
+
51
+ if "text/html" not in content_type.lower():
52
+ # Non-HTML: no main-content concept; decoded text alone is LOW.
53
+ return Quality.LOW
54
+
55
+ if lossy:
56
+ return Quality.LOW
57
+
58
+ if not has_body_text:
59
+ # Only boilerplate/links/frames/forms were found.
60
+ return Quality.LOW
61
+
62
+ if found_main and len(text) >= RICH_TEXT_FLOOR:
63
+ return Quality.HIGH
64
+
65
+ return Quality.MEDIUM
66
+
67
+
68
+ def _resolve_text_format_quality(quality_str: str) -> Quality:
69
+ """Map text format quality string to Quality enum."""
70
+ mapping = {
71
+ "high": Quality.HIGH,
72
+ "medium": Quality.MEDIUM,
73
+ "low": Quality.LOW,
74
+ "failed": Quality.FAILED,
75
+ }
76
+ return mapping.get(quality_str, Quality.FAILED)
77
+
78
+
79
+ def _apply_text_format(
80
+ fmt: dict[str, object],
81
+ metadata: dict[str, str],
82
+ flags: list[str],
83
+ ) -> tuple[
84
+ str,
85
+ str,
86
+ list[NormalizedUrl],
87
+ list,
88
+ list,
89
+ list[list[list[str]]],
90
+ bool,
91
+ Quality,
92
+ ]:
93
+ """Apply a text-format result dict to local variables.
94
+
95
+ Shared by CSV/JSON/XML branches to avoid duplicating the same
96
+ dict-unpacking logic three times.
97
+ """
98
+ metadata.update(cast(dict[str, str], fmt["metadata"]))
99
+ text = cast(str, fmt["text"])
100
+ markdown = cast(str, fmt["markdown"])
101
+ links = cast(list[NormalizedUrl], fmt["links"])
102
+ frames = cast(list[dict[str, str]], fmt["frames"])
103
+ forms = cast(list[dict[str, str]], fmt["forms"])
104
+ tables = cast(list[list[list[str]]], fmt["tables"])
105
+ truncated = cast(bool, fmt["truncated"])
106
+ flags.extend(cast(list[str], fmt.get("quality_flags", [])))
107
+ quality = _resolve_text_format_quality(cast(str, fmt.get("_quality", "failed")))
108
+ return text, markdown, links, frames, forms, tables, truncated, quality
109
+
110
+
111
+ def _empty_extraction(
112
+ *,
113
+ quality: Quality,
114
+ flag: str,
115
+ flags: list[str],
116
+ metadata: dict[str, str],
117
+ charset: str | None,
118
+ ) -> Extraction:
119
+ """Build an empty Extraction with the given quality and flag."""
120
+ return Extraction(
121
+ quality=quality,
122
+ flags=flags + [flag],
123
+ metadata=metadata,
124
+ text="",
125
+ markdown="",
126
+ links=[],
127
+ frames=[],
128
+ forms=[],
129
+ tables=[],
130
+ truncated=False,
131
+ charset=charset,
132
+ )
133
+
134
+
135
+ def extract(
136
+ content: bytes,
137
+ content_type: str,
138
+ *,
139
+ url: str | None = None,
140
+ limits: ExtractLimits = DEFAULT_LIMITS,
141
+ ) -> Extraction:
142
+ """Extract bounded, deterministic output. Refuses oversized input.
143
+
144
+ Never raises on hostile input; returns quality="failed" instead."""
145
+ if len(content) > limits.max_input_bytes:
146
+ return Extraction(quality=Quality.FAILED, flags=["input_refused"])
147
+ metadata: dict[str, str] = {}
148
+ if content_type:
149
+ metadata["content_type"] = content_type
150
+ if url is not None:
151
+ metadata["url"] = url
152
+
153
+ ctype = content_type.lower()
154
+ container_types = (
155
+ "application/msword", # .doc legacy binary — refused with flag
156
+ )
157
+ is_office = (
158
+ ctype == "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
159
+ or ctype == "application/vnd.openxmlformats-officedocument.presentationml.presentation"
160
+ or ctype == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
161
+ or ctype == "application/epub+zip"
162
+ )
163
+
164
+ # Binary containers (PDF, DOC) skip the charset ladder — decode_content
165
+ # would emit charset_lossy on non-text bytes.
166
+ flags: list[str] = []
167
+ if ctype == "application/pdf":
168
+ # PDF tier-1 text sweep; below the quality floor returns empty + low.
169
+ fmt = extract_pdf(content, limits)
170
+ (
171
+ pdf_text,
172
+ pdf_markdown,
173
+ pdf_links,
174
+ pdf_frames,
175
+ pdf_forms,
176
+ pdf_tables,
177
+ pdf_truncated,
178
+ pdf_quality,
179
+ ) = _apply_text_format(fmt, metadata, flags)
180
+ return Extraction(
181
+ quality=pdf_quality,
182
+ flags=flags,
183
+ metadata=metadata,
184
+ text=pdf_text,
185
+ markdown=pdf_markdown,
186
+ links=pdf_links,
187
+ frames=pdf_frames,
188
+ forms=pdf_forms,
189
+ tables=pdf_tables,
190
+ truncated=pdf_truncated,
191
+ charset=None,
192
+ links_refused=0,
193
+ )
194
+ if ctype in container_types:
195
+ charset: str | None = None
196
+ return _empty_extraction(
197
+ quality=Quality.LOW,
198
+ flag=f"{ctype.split('/')[-1]}_unsupported",
199
+ flags=flags,
200
+ metadata=metadata,
201
+ charset=charset,
202
+ )
203
+ # Office containers (DOCX/PPTX/XLSX/EPUB) extracted via zip+XML.
204
+ if is_office:
205
+ fmt = extract_office(content, content_type, limits)
206
+ (
207
+ of_text,
208
+ of_markdown,
209
+ of_links,
210
+ of_frames,
211
+ of_forms,
212
+ of_tables,
213
+ of_truncated,
214
+ of_quality,
215
+ ) = _apply_text_format(fmt, metadata, flags)
216
+ return Extraction(
217
+ quality=of_quality,
218
+ flags=flags,
219
+ metadata=metadata,
220
+ text=of_text,
221
+ markdown=of_markdown,
222
+ links=of_links,
223
+ frames=of_frames,
224
+ forms=of_forms,
225
+ tables=of_tables,
226
+ truncated=of_truncated,
227
+ charset=None,
228
+ links_refused=0,
229
+ )
230
+ # Truly unsupported type: refuse without decode.
231
+ if not (
232
+ ctype == "text/html"
233
+ or ctype.startswith("text/html;")
234
+ or ctype == "text/csv"
235
+ or ctype.startswith("text/csv;")
236
+ or ctype == "application/csv"
237
+ or ctype == "application/json"
238
+ or ctype.startswith("application/json;")
239
+ or ctype == "application/jsonl"
240
+ or ctype.startswith("application/jsonl;")
241
+ or ctype == "text/json"
242
+ or ctype == "application/xml"
243
+ or ctype.startswith("application/xml;")
244
+ or ctype == "text/xml"
245
+ or ctype.startswith("text/xml;")
246
+ or ctype.startswith("text/")
247
+ ):
248
+ return _empty_extraction(
249
+ quality=Quality.FAILED,
250
+ flag="format_refused",
251
+ flags=[],
252
+ metadata=metadata,
253
+ charset=None,
254
+ )
255
+
256
+ text, charset, flags = decode_content(content, content_type)
257
+ if charset:
258
+ metadata["charset"] = charset
259
+
260
+ markdown = ""
261
+ links: list = []
262
+ frames: list = []
263
+ forms: list = []
264
+ tables: list[list[list[str]]] = []
265
+ truncated = False
266
+ quality = Quality.FAILED
267
+ links_refused = 0
268
+
269
+ is_html = ctype == "text/html" or ctype.startswith("text/html;")
270
+ is_csv = ctype == "text/csv" or ctype.startswith("text/csv;") or ctype == "application/csv"
271
+ is_json = (
272
+ ctype == "application/json"
273
+ or ctype.startswith("application/json;")
274
+ or ctype == "application/jsonl"
275
+ or ctype.startswith("application/jsonl;")
276
+ or ctype == "text/json"
277
+ )
278
+ is_xml = (
279
+ ctype == "application/xml"
280
+ or ctype.startswith("application/xml;")
281
+ or ctype == "text/xml"
282
+ or ctype.startswith("text/xml;")
283
+ )
284
+ if is_html:
285
+ policy = HostPolicy.unrestricted()
286
+ html = extract_html(text, limits, base_url=url, policy=policy)
287
+ metadata.update(html["metadata"])
288
+ text = html["text"]
289
+ markdown = html["markdown"]
290
+ links = html["links"]
291
+ frames = html["frames"]
292
+ forms = html["forms"]
293
+ tables = html["tables"]
294
+ truncated = html["truncated"]
295
+ if html["links_refused"] > 0:
296
+ flags.append("links_refused")
297
+ flags.extend(html.get("flags", []))
298
+ links_refused = html["links_refused"]
299
+ quality = _resolve_quality(
300
+ content_type=content_type,
301
+ text=text,
302
+ flags=flags,
303
+ found_main=html["found_main"],
304
+ html_links=links,
305
+ html_frames=frames,
306
+ html_forms=forms,
307
+ html_tables=tables,
308
+ )
309
+ elif is_csv or is_json or is_xml:
310
+ fmt = extract_text_format(text, content_type, limits)
311
+ (
312
+ text,
313
+ markdown,
314
+ links,
315
+ frames,
316
+ forms,
317
+ tables,
318
+ truncated,
319
+ quality,
320
+ ) = _apply_text_format(fmt, metadata, flags)
321
+ elif ctype.startswith("text/"):
322
+ # Plain text: no format extractor ran. Empty → FAILED, else LOW.
323
+ quality = Quality.LOW if text.strip() else Quality.FAILED
324
+
325
+ return Extraction(
326
+ quality=quality,
327
+ flags=flags,
328
+ metadata=metadata,
329
+ text=text,
330
+ markdown=markdown,
331
+ links=links,
332
+ frames=frames,
333
+ forms=forms,
334
+ tables=tables,
335
+ truncated=truncated,
336
+ charset=charset,
337
+ links_refused=links_refused,
338
+ )
@@ -0,0 +1,49 @@
1
+ """Shared HTML parse helpers over html.parser attr lists."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from html import parser as html_parser
6
+
7
+ from safe_url import HostPolicy, NormalizedUrl, admit_or_none
8
+
9
+ from crawlutron_extract.limits import ExtractLimits
10
+
11
+ __all__ = ["resolve_or_admit", "tag_attrs"]
12
+
13
+
14
+ class _HtmlParser(html_parser.HTMLParser):
15
+ """Base parser for single-pass HTML discovery: charrefs converted."""
16
+
17
+ def __init__(self) -> None:
18
+ super().__init__(convert_charrefs=True)
19
+
20
+
21
+ def resolve_or_admit(href: str, target: NormalizedUrl | None) -> NormalizedUrl | None:
22
+ """Resolve href against target, or admit it outright when there is no base URL."""
23
+ if target is not None:
24
+ return target.resolve(href)
25
+ return admit_or_none(href, policy=HostPolicy.unrestricted())
26
+
27
+
28
+ def _prepare(
29
+ html: bytes | str,
30
+ source_url: str | NormalizedUrl | None,
31
+ limits: ExtractLimits | None = None,
32
+ ) -> tuple[str, NormalizedUrl | None, ExtractLimits]:
33
+ """Decode html, admit the source URL, and apply the default limits."""
34
+ if limits is None:
35
+ limits = ExtractLimits()
36
+ text = html.decode("utf-8", "replace") if isinstance(html, bytes) else html
37
+ base: NormalizedUrl | None = None
38
+ if isinstance(source_url, NormalizedUrl):
39
+ base = source_url
40
+ elif source_url is not None:
41
+ admitted = admit_or_none(source_url, policy=HostPolicy.unrestricted())
42
+ if admitted is not None:
43
+ base = admitted
44
+ return text, base, limits
45
+
46
+
47
+ def tag_attrs(attrs: list[tuple[str, str | None]]) -> dict[str, str]:
48
+ """html.parser's (name, value) pairs as a string dict, values normalised."""
49
+ return {name: (value or "") for name, value in attrs}
@@ -0,0 +1,84 @@
1
+ """Shared tokenizer helpers and the shared ``SelectorError`` for the CSS and
2
+ XPath selector parsers, kept in one module to avoid jscpd clones.
3
+ """
4
+
5
+ from __future__ import annotations
6
+
7
+ from typing import Never
8
+
9
+
10
+ class SelectorError(Exception):
11
+ """Raised when a selector uses syntax outside the supported subset."""
12
+
13
+ def __init__(self, message: str, selector: str = "", token: str | None = None):
14
+ sel = f" (selector: {selector!r})" if selector else ""
15
+ tail = f" (token: {token!r})" if token is not None else ""
16
+ super().__init__(f"{message}{sel}{tail}")
17
+ self.selector = selector
18
+ self.expression = selector
19
+ self.token = token
20
+
21
+
22
+ def _peek(tokens: list[tuple[str, str]], pos: int) -> tuple[str, str] | None:
23
+ if 0 <= pos < len(tokens):
24
+ return tokens[pos]
25
+ return None
26
+
27
+
28
+ def _consume(
29
+ tokens: list[tuple[str, str]], pos: int, expected: str | None = None
30
+ ) -> tuple[tuple[str, str], int]:
31
+ if pos >= len(tokens):
32
+ raise SelectorError("Unexpected end of expression")
33
+ tok = tokens[pos]
34
+ if expected and tok[0] != expected:
35
+ raise SelectorError(f"Expected {expected}, got {tok[0]}", token=tok[1])
36
+ return tok, pos + 1
37
+
38
+
39
+ def _skip_ws(tokens: list[tuple[str, str]], pos: int) -> int:
40
+ tok = _peek(tokens, pos)
41
+ while tok and tok[0] == "WS":
42
+ _, pos = _consume(tokens, pos, "WS")
43
+ tok = _peek(tokens, pos)
44
+ return pos
45
+
46
+
47
+ def _emit_char(tokens: list[tuple[str, str]], kind: str, char: str) -> None:
48
+ tokens.append((kind, char))
49
+
50
+
51
+ def _scan_ident(text: str, i: int, n: int, extra: str = "-_.") -> int:
52
+ j = i
53
+ while j < n and (text[j].isalnum() or text[j] in extra):
54
+ j += 1
55
+ return j
56
+
57
+
58
+ FUNC_NAMES = frozenset({"contains", "starts-with", "normalize-space", "concat"})
59
+
60
+
61
+ def _is_func_call(tok: tuple[str, str] | None) -> bool:
62
+ return tok is not None and tok[0] == "NAME" and tok[1] in FUNC_NAMES
63
+
64
+
65
+ def _consume_parens(tokens: list[tuple[str, str]], pos: int) -> int:
66
+ _, pos = _consume(tokens, pos, "LPAREN")
67
+ _, pos = _consume(tokens, pos, "RPAREN")
68
+ return pos
69
+
70
+
71
+ def _consume_attr(tokens: list[tuple[str, str]], pos: int) -> int:
72
+ _, pos = _consume(tokens, pos, "AT")
73
+ return pos
74
+
75
+
76
+ def _parse_string_literal(
77
+ tok: tuple[str, str], tokens: list[tuple[str, str]], pos: int
78
+ ) -> tuple[tuple[str, ...], int]:
79
+ _, pos = _consume(tokens, pos, "STRING")
80
+ return ("STRING_LITERAL", tok[1]), pos
81
+
82
+
83
+ def _unexpected_end(tokens: list[tuple[str, str]], pos: int, message: str, expression: str) -> Never:
84
+ raise SelectorError(message, expression, None)
@@ -0,0 +1,94 @@
1
+ """Charset detection and lossy-safe decoding.
2
+
3
+ BOM -> Content-Type -> <meta charset> -> strict UTF-8 -> lossy. Never raises.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import re
9
+
10
+ __all__ = ["decode_content"]
11
+
12
+ # BOM -> charset, longest BOM first so UTF-8 (3 bytes) wins over UTF-16 (2 bytes).
13
+ _BOM_CHARS = (
14
+ (b"\xef\xbb\xbf", "utf-8"),
15
+ (b"\xff\xfe", "utf-16-le"),
16
+ (b"\xfe\xff", "utf-16-be"),
17
+ )
18
+
19
+ # Bounded meta-charset scan: first 8 KiB only, no full HTML parse.
20
+ _META_CHARSET_RE = re.compile(
21
+ r'<meta[^>]+charset\s*=\s*["\']?([^\s"\']+)',
22
+ re.IGNORECASE,
23
+ )
24
+ _META_CONTENT_TYPE_RE = re.compile(
25
+ r'<meta[^>]+http-equiv\s*=\s*["\']?content-type["\']?'
26
+ r'[^>]+content\s*=\s*["\']?[^;\s"]*;\s*charset\s*=\s*([^\s"\']+)',
27
+ re.IGNORECASE,
28
+ )
29
+
30
+ _META_SCAN_BYTES = 8192
31
+
32
+
33
+ def _strip_bom(content: bytes) -> tuple[bytes, str | None]:
34
+ """Strip a leading BOM, returning (body, charset) or (content, None)."""
35
+ for bom, charset in _BOM_CHARS:
36
+ if content.startswith(bom):
37
+ return content[len(bom) :], charset
38
+ return content, None
39
+
40
+
41
+ def _parse_content_type_charset(content_type: str) -> str | None:
42
+ """Extract charset= from Content-Type, with or without quotes."""
43
+ if not content_type:
44
+ return None
45
+ match = re.search(r";\s*charset\s*=\s*[\"']?([^\s\";]+)", content_type, re.IGNORECASE)
46
+ return match.group(1).strip().lower() if match else None
47
+
48
+
49
+ def _scan_meta_charset(content: bytes) -> str | None:
50
+ """Scan the first 8 KiB for an HTML meta charset declaration."""
51
+ head = content[:_META_SCAN_BYTES]
52
+ text = head.decode("utf-8", errors="replace")
53
+ for pattern in (_META_CHARSET_RE, _META_CONTENT_TYPE_RE):
54
+ match = pattern.search(text)
55
+ if match:
56
+ return match.group(1).strip().lower()
57
+ return None
58
+
59
+
60
+ def _detect_charset(content: bytes, content_type: str) -> str:
61
+ """Resolve charset by priority: BOM -> Content-Type -> meta -> UTF-8."""
62
+ _, bom_charset = _strip_bom(content)
63
+ if bom_charset:
64
+ return bom_charset
65
+
66
+ declared = _parse_content_type_charset(content_type)
67
+ if declared:
68
+ return declared
69
+
70
+ if "text/html" in content_type.lower():
71
+ meta = _scan_meta_charset(content)
72
+ if meta:
73
+ return meta
74
+
75
+ return "utf-8"
76
+
77
+
78
+ def decode_content(content: bytes, content_type: str) -> tuple[str, str, list[str]]:
79
+ """Decode content, never raising: returns (text, charset, flags).
80
+
81
+ Decode failure is reported only through the charset_lossy flag.
82
+ """
83
+ flags: list[str] = []
84
+ body, _bom_charset = _strip_bom(content)
85
+
86
+ charset = _detect_charset(content, content_type)
87
+
88
+ try:
89
+ text = body.decode(charset)
90
+ except UnicodeDecodeError, LookupError:
91
+ flags.append("charset_lossy")
92
+ text = body.decode("utf-8", errors="replace")
93
+
94
+ return text, charset, flags