crawlutron-extract 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. crawlutron_extract-0.5.0/.gitignore +38 -0
  2. crawlutron_extract-0.5.0/PKG-INFO +233 -0
  3. crawlutron_extract-0.5.0/README.md +223 -0
  4. crawlutron_extract-0.5.0/bench/RESULTS.md +19 -0
  5. crawlutron_extract-0.5.0/bench/bench_extract.py +101 -0
  6. crawlutron_extract-0.5.0/pyproject.toml +20 -0
  7. crawlutron_extract-0.5.0/src/crawlutron_extract/__init__.py +36 -0
  8. crawlutron_extract-0.5.0/src/crawlutron_extract/_extract.py +338 -0
  9. crawlutron_extract-0.5.0/src/crawlutron_extract/_html_common.py +49 -0
  10. crawlutron_extract-0.5.0/src/crawlutron_extract/_select_common.py +84 -0
  11. crawlutron_extract-0.5.0/src/crawlutron_extract/charset.py +94 -0
  12. crawlutron_extract-0.5.0/src/crawlutron_extract/css_select.py +447 -0
  13. crawlutron_extract-0.5.0/src/crawlutron_extract/html_diagnostics.py +72 -0
  14. crawlutron_extract-0.5.0/src/crawlutron_extract/html_dom.py +114 -0
  15. crawlutron_extract-0.5.0/src/crawlutron_extract/html_forms_frames.py +109 -0
  16. crawlutron_extract-0.5.0/src/crawlutron_extract/html_links.py +84 -0
  17. crawlutron_extract-0.5.0/src/crawlutron_extract/html_meta.py +160 -0
  18. crawlutron_extract-0.5.0/src/crawlutron_extract/html_pass.py +212 -0
  19. crawlutron_extract-0.5.0/src/crawlutron_extract/html_tables.py +213 -0
  20. crawlutron_extract-0.5.0/src/crawlutron_extract/html_text.py +211 -0
  21. crawlutron_extract-0.5.0/src/crawlutron_extract/js_endpoints.py +55 -0
  22. crawlutron_extract-0.5.0/src/crawlutron_extract/limits.py +59 -0
  23. crawlutron_extract-0.5.0/src/crawlutron_extract/links.py +109 -0
  24. crawlutron_extract-0.5.0/src/crawlutron_extract/models.py +39 -0
  25. crawlutron_extract-0.5.0/src/crawlutron_extract/office.py +360 -0
  26. crawlutron_extract-0.5.0/src/crawlutron_extract/pagination.py +50 -0
  27. crawlutron_extract-0.5.0/src/crawlutron_extract/pdf_pass.py +351 -0
  28. crawlutron_extract-0.5.0/src/crawlutron_extract/py.typed +0 -0
  29. crawlutron_extract-0.5.0/src/crawlutron_extract/schema.py +248 -0
  30. crawlutron_extract-0.5.0/src/crawlutron_extract/sitemap.py +59 -0
  31. crawlutron_extract-0.5.0/src/crawlutron_extract/text_format.py +281 -0
  32. crawlutron_extract-0.5.0/src/crawlutron_extract/xpath_eval.py +143 -0
  33. crawlutron_extract-0.5.0/src/crawlutron_extract/xpath_select.py +283 -0
  34. crawlutron_extract-0.5.0/tests/conftest.py +139 -0
  35. crawlutron_extract-0.5.0/tests/fixtures/degraded_table.pdf +39 -0
  36. crawlutron_extract-0.5.0/tests/fixtures/generate_xpath_golden.py +97 -0
  37. crawlutron_extract-0.5.0/tests/fixtures/jo.pdf +0 -0
  38. crawlutron_extract-0.5.0/tests/fixtures/sample_table.pdf +39 -0
  39. crawlutron_extract-0.5.0/tests/fixtures/simple2.pdf +78 -0
  40. crawlutron_extract-0.5.0/tests/fixtures/xpath_golden.json +148 -0
  41. crawlutron_extract-0.5.0/tests/fixtures/zen_of_python_corrupted.pdf +0 -0
  42. crawlutron_extract-0.5.0/tests/golden/123233_charts.xlsx +0 -0
  43. crawlutron_extract-0.5.0/tests/golden/1_NoIden.xlsx +0 -0
  44. crawlutron_extract-0.5.0/tests/golden/49386-null_dates.pptx +0 -0
  45. crawlutron_extract-0.5.0/tests/golden/52288.docx +0 -0
  46. crawlutron_extract-0.5.0/tests/golden/55733.docx +0 -0
  47. crawlutron_extract-0.5.0/tests/golden/LICENSES.md +49 -0
  48. crawlutron_extract-0.5.0/tests/golden/encoding_test-yahoo-jp.dat +10 -0
  49. crawlutron_extract-0.5.0/tests/golden/encoding_tests2.dat +115 -0
  50. crawlutron_extract-0.5.0/tests/golden/sampleOneByteIdentityEncode.pdf +0 -0
  51. crawlutron_extract-0.5.0/tests/golden/simple1.pdf +68 -0
  52. crawlutron_extract-0.5.0/tests/golden/simple3.pdf +87 -0
  53. crawlutron_extract-0.5.0/tests/golden/test_pattern_colors.pdf +55 -0
  54. crawlutron_extract-0.5.0/tests/golden/tokenizer_escapeFlag.test +36 -0
  55. crawlutron_extract-0.5.0/tests/golden/tokenizer_test1.test +353 -0
  56. crawlutron_extract-0.5.0/tests/golden/tokenizer_unicodeCharsProblematic.test +41 -0
  57. crawlutron_extract-0.5.0/tests/golden/tokenizer_xmlViolation.test +20 -0
  58. crawlutron_extract-0.5.0/tests/test_bench_timing.py +25 -0
  59. crawlutron_extract-0.5.0/tests/test_charset.py +167 -0
  60. crawlutron_extract-0.5.0/tests/test_css_select.py +53 -0
  61. crawlutron_extract-0.5.0/tests/test_detect_pagination.py +40 -0
  62. crawlutron_extract-0.5.0/tests/test_extract_discovered_links.py +87 -0
  63. crawlutron_extract-0.5.0/tests/test_extract_js_endpoints.py +48 -0
  64. crawlutron_extract-0.5.0/tests/test_extract_sitemap_locs.py +74 -0
  65. crawlutron_extract-0.5.0/tests/test_golden_corpus.py +178 -0
  66. crawlutron_extract-0.5.0/tests/test_html_diagnostics.py +62 -0
  67. crawlutron_extract-0.5.0/tests/test_html_forms_frames.py +120 -0
  68. crawlutron_extract-0.5.0/tests/test_html_metadata.py +141 -0
  69. crawlutron_extract-0.5.0/tests/test_html_pass.py +499 -0
  70. crawlutron_extract-0.5.0/tests/test_office.py +499 -0
  71. crawlutron_extract-0.5.0/tests/test_pdf_pass.py +123 -0
  72. crawlutron_extract-0.5.0/tests/test_pdf_tier2.py +67 -0
  73. crawlutron_extract-0.5.0/tests/test_schema.py +298 -0
  74. crawlutron_extract-0.5.0/tests/test_skeleton.py +126 -0
  75. crawlutron_extract-0.5.0/tests/test_text_format.py +153 -0
  76. crawlutron_extract-0.5.0/tests/test_xpath_adversarial.py +92 -0
  77. crawlutron_extract-0.5.0/tests/test_xpath_golden.py +58 -0
@@ -0,0 +1,38 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ .pytest_cache/
4
+ report/
5
+ # Build artifacts from `python -m build` / editable installs (#26).
6
+ dist/
7
+ build/
8
+ # Both spellings: a linked worktree reaches the venv through a `.venv` symlink,
9
+ # which the trailing-slash form does not match -- and a committed symlink into
10
+ # one checkout's venv is worse than no venv at all.
11
+ .venv/
12
+ .venv
13
+ .dynamic-task-ledger.json
14
+ *.egg-info/
15
+ .cache/
16
+ .mypy_cache/
17
+ .ruff_cache/
18
+ .gate_fanout/
19
+ .coverage
20
+ htmlcov/
21
+ # Session-analysis residue written by the cortexkit tooling. It is ephemeral
22
+ # per-session output, not project content, and its own managed block already
23
+ # declares `magic-context/` untracked -- the historian XML under it was
24
+ # committed anyway, which is how a directory full of transcript digests ended
25
+ # up in the tree this service is extracted from.
26
+ .cortexkit/
27
+ logs/
28
+ cache/
29
+ .env
30
+ tmp/
31
+ *.log
32
+ .DS_Store
33
+
34
+ .coverage.*
35
+ .opencode
36
+ .hardstand/
37
+
38
+
@@ -0,0 +1,233 @@
1
+ Metadata-Version: 2.5
2
+ Name: crawlutron-extract
3
+ Version: 0.5.0
4
+ Summary: Content extraction: HTML, office containers, text formats, PDF tier 1, bounded and deterministic
5
+ License-Expression: MIT
6
+ Requires-Python: >=3.14
7
+ Requires-Dist: crawlutron-safe-url
8
+ Requires-Dist: pdfminer-six==20260107
9
+ Description-Content-Type: text/markdown
10
+
11
+ # crawlutron-extract
12
+
13
+ Bounded, deterministic, never-raise content extraction. Converts a byte stream plus a
14
+ content type into a single `Extraction` payload — capped text/markdown, links, frames,
15
+ forms, tables, schema records, a quality grade, and degradation flags — for HTML, office
16
+ containers (DOCX/XLSX/PPTX/EPUB), text formats (CSV/JSON/XML), and a PDF tier-1 text
17
+ sweep via pdfminer.six.
18
+
19
+ There is exactly one entry point, `extract()`. Everything else is derived from its
20
+ `Extraction` result.
21
+
22
+ ## The single entry point
23
+
24
+ ```python
25
+ from crawlutron_extract import extract, DEFAULT_LIMITS
26
+ from crawlutron_extract.models import Extraction
27
+
28
+ result: Extraction = extract(
29
+ content: bytes,
30
+ content_type: str,
31
+ *,
32
+ url: str | None = None,
33
+ limits: ExtractLimits = DEFAULT_LIMITS,
34
+ )
35
+ ```
36
+
37
+ `extract` is the single entry point for every format. It:
38
+
39
+ * never raises — hostile or malformed input yields `quality="failed"` plus a flag,
40
+ * refuses input above `limits.max_input_bytes` (`input_refused`),
41
+ * dispatches by content type to the HTML, office, PDF, or text-format path,
42
+ * returns a fresh, deterministic `Extraction` on every call (two calls on the same bytes
43
+ are byte-identical).
44
+
45
+ Links are admitted through `crawlutron-safe-url` (`NormalizedUrl`), syntax-checked and
46
+ refused when unsafe; DNS-level pinning is the caller's job, not the extractor's.
47
+
48
+ ## Public types
49
+
50
+ All public symbols live in `crawlutron_extract` (see `__all__`).
51
+
52
+ **`Extraction`** — the bounded, deterministic result payload:
53
+ * `text: str` — extracted body text, capped at `max_text_chars`
54
+ * `markdown: str` — markdown rendering, capped at `max_markdown_chars`
55
+ * `charset: str | None` — detected charset for text/HTML
56
+ * `metadata: dict[str, str]` — content type, charset, url, container/format hints
57
+ * `links: list[NormalizedUrl]` — admitted links, capped at `max_links`
58
+ * `frames: list[dict[str, str]]` — frames, capped at `max_frames`
59
+ * `forms: list[dict]` — forms, capped at `max_forms`
60
+ * `tables: list[list[list[str]]]` — tables as row/cell grids, capped at `max_tables`
61
+ * `quality: Quality` — `HIGH` / `MEDIUM` / `LOW` / `FAILED`
62
+ * `flags: list[str]` — degradation signals, never data loss
63
+ * `truncated: bool` — any output list was capped
64
+ * `links_refused: int` — link count refused by the URL policy
65
+ * `records: tuple[Mapping[str, object], ...]` — schema records (schema path only)
66
+
67
+ **`Quality`** (`StrEnum`) — closed ladder: `HIGH`, `MEDIUM`, `LOW`, `FAILED`.
68
+
69
+ **`ExtractLimits`** (frozen dataclass) — every cap applied to an extraction; see
70
+ "Limits and defaults" below. `DEFAULT_LIMITS` is the module-level instance.
71
+
72
+ **Schema types** (`crawlutron_extract.schema`):
73
+ * `FieldSpec(name, selector, kind=FieldKind.TEXT, attribute=None, required=False, many=False)`
74
+ * `FieldKind` — `TEXT`, `MARKDOWN`, `ATTR`, `LINK`, `NUMBER`, `INTEGER`
75
+ * `Schema(record: str | None, fields: tuple[FieldSpec, ...])`
76
+ * `extract_records(html, schema, *, url=None, limits=None) -> Extraction`
77
+
78
+ **Selectors**: `validate_css(selector)`, `validate_xpath(expr)`, both raise
79
+ `SelectorError` when an expression is outside the supported subset.
80
+
81
+ Other public doors: `extract_discovered_links`, `extract_js_endpoints`,
82
+ `extract_sitemap_locs`, `html_diagnostics`, `detect_pagination`, `LinkRecord`.
83
+
84
+ ## Schema language with a worked example
85
+
86
+ The schema path extracts structured records from HTML using a closed CSS-selector
87
+ subset. A `Schema` pairs an optional record selector with a tuple of `FieldSpec`s. When
88
+ `record` is `None` the whole document is one record.
89
+
90
+ ```python
91
+ from crawlutron_extract import DEFAULT_LIMITS, extract_records
92
+ from crawlutron_extract.schema import FieldKind, FieldSpec, Schema
93
+
94
+ schema = Schema(
95
+ record="article.product",
96
+ fields=(
97
+ FieldSpec("title", "h1", FieldKind.TEXT),
98
+ FieldSpec("price", "span.price", FieldKind.NUMBER),
99
+ FieldSpec("link", "a", FieldKind.LINK, many=True),
100
+ FieldSpec("in_stock", ".stock", FieldKind.TEXT),
101
+ ),
102
+ )
103
+ result = extract_records(html_bytes, schema, url="https://example.com/", limits=DEFAULT_LIMITS)
104
+ # result.records -> tuple of dicts like {"title": "...", "price": 12.5, "link": (...), ...}
105
+ ```
106
+
107
+ `FieldKind.LINK` resolves and admits each `href` through `crawlutron-safe-url`; refused
108
+ links are dropped. `NUMBER`/`INTEGER` coerce text; a missing required field sets
109
+ `schema_required_missing`. Output is bounded by `max_records` and `max_fields_per_record`.
110
+
111
+ ## CSS subset (closed — refused outside)
112
+
113
+ `validate_css` accepts only this subset; anything else raises `SelectorError`:
114
+
115
+ * type selector (`div`), universal (`*`)
116
+ * class (`.c`) and id (`#i`)
117
+ * attribute presence (`[a]`) and equality / prefix / suffix / contains
118
+ (`[a='v']`, `[a^='v']`, `[a$='v']`, `[a*='v']`)
119
+ * combinators: descendant (whitespace), child (`>`), adjacent (`+`)
120
+ * pseudo-classes: `:first-child`, `:last-child`, `:nth-child(n)`, `:not(s)`
121
+ * comma-separated selector lists
122
+
123
+ Refused (outside the subset, raises `SelectorError`): namespaces, pseudo-elements (`::`),
124
+ `:hover`/`:nth-of-type`/`:has`/`:is`/`:where` and any pseudo-class not listed, attribute
125
+ operators other than `= ^= $= *=`, and `:not()` containing a non-simple selector. The
126
+ subset is closed by design: if `validate_css` accepts it, the evaluator can match it; if
127
+ the evaluator cannot match it, `validate_css` rejects it.
128
+
129
+ ## XPath subset (closed — refused outside)
130
+
131
+ `validate_xpath` accepts only this subset; anything else raises `SelectorError` naming the
132
+ offending token:
133
+
134
+ * location steps: `/`, `//`, `.//`
135
+ * node test: `*`, a tag name, `text()`, `@attr`
136
+ * predicates: `[n]`, `[last()]`, `[@a]`, `[@a='v']`
137
+ * functions: `contains()`, `concat()`, `normalize-space()`, `starts-with()`
138
+
139
+ Refused (outside the subset): axes (`self::`, `parent::`, `child::`, `descendant::`,
140
+ `attribute::` and any other), any function not listed, any predicate form not listed, and
141
+ any unknown character or token. The subset is closed: `validate_xpath` rejects everything
142
+ the bounded evaluator cannot execute.
143
+
144
+ ## Limits and defaults
145
+
146
+ `ExtractLimits` (frozen dataclass), defaults shown. `max_input_bytes` refuses oversized
147
+ input before any decode; every list output is capped and sets `truncated`.
148
+
149
+ | Limit | Default | Meaning |
150
+ |---|---|---|
151
+ | `max_input_bytes` | 50,000,000 | refuse input larger than this (50 MB) |
152
+ | `max_text_chars` | 100,000 | cap on extracted `text` |
153
+ | `max_markdown_chars` | 200,000 | cap on extracted `markdown` |
154
+ | `max_links` | 1,000 | cap on admitted links |
155
+ | `max_tables` | 50 | cap on tables |
156
+ | `max_table_rows` | 1,000 | rows per table |
157
+ | `max_table_cols` | 100 | columns per table |
158
+ | `max_frames` | 100 | cap on frames |
159
+ | `max_forms` | 100 | cap on forms |
160
+ | `max_dom_nodes` | 50,000 | DOM node cap (HTML/schema path) |
161
+ | `max_dom_depth` | 50 | DOM depth cap |
162
+ | `max_records` | 10,000 | schema records cap |
163
+ | `max_fields_per_record` | 100 | fields per schema record cap |
164
+ | `max_zip_entries` | 10,000 | office container entry cap |
165
+ | `max_zip_ratio` | 1,000 | office container compression-bomb ratio cap |
166
+
167
+ `RICH_TEXT_FLOOR = 200` is the minimum body-text length for `quality="high"` (requires a
168
+ main-content element and a non-lossy decode).
169
+
170
+ ## Quality ladder
171
+
172
+ Closed vocabulary, evaluated by `_resolve_quality`:
173
+
174
+ | Grade | Meaning |
175
+ |---|---|
176
+ | `HIGH` | main-content element found, body text ≥ `RICH_TEXT_FLOOR`, decode not lossy |
177
+ | `MEDIUM` | body text present, but below the floor or a lossy decode |
178
+ | `LOW` | decoded text or structural content (links/forms/tables) present, no body text / lossy |
179
+ | `FAILED` | no extractable content, refused format, or input refused |
180
+
181
+ A clean non-HTML document with decoded text is `LOW` (no main-content concept). An empty
182
+ result is never above `LOW`.
183
+
184
+ ## Flags vocabulary (degradation, not failure)
185
+
186
+ Flags describe degradation, never data loss. `extract` never returns partial data silently:
187
+ every flag is paired with the result it qualifies, and `truncated` is set whenever a cap
188
+ bit. Representative flags:
189
+
190
+ * `charset_lossy` — text had to be lossily re-encoded
191
+ * `links_refused` — one or more links refused by the URL policy
192
+ * `input_refused` — input exceeded `max_input_bytes`
193
+ * `format_refused` — content type not supported
194
+ * `selector_invalid` / `schema_required_missing` — schema selector or required field issue
195
+ * `records_truncated` / `fields_truncated` — a schema cap was hit
196
+ * `schema_dom_cap` — the DOM cap prevented schema extraction
197
+ * `pdf_encoding_partial` — PDF text sweep could not recover the full encoding
198
+ * `table_grid_degraded` / `nested_table_flattened` — a table was normalized, not dropped
199
+
200
+ Text is never discarded because markdown cannot express a shape: nested tables flatten into
201
+ the enclosing cell (pipes escaped, `nested_table_flattened` set), `colspan` repeats content
202
+ and `rowspan` carries it down, and the grid stays rectangular.
203
+
204
+ ## Hard dependency set
205
+
206
+ `crawlutron-extract` ships no extras and is not stdlib-only. The hard dependencies are
207
+ imported plainly — a broken install fails at import, there is no lazy/fallback path:
208
+
209
+ * `crawlutron-safe-url` — `NormalizedUrl` and `HostPolicy`; its own closure is also hard:
210
+ * `regex` — bounded matcher
211
+ * `publicsuffixlist` — public suffix list
212
+ * `pdfminer.six` — PDF tier-1 text sweep
213
+
214
+ Credentials and URLs are object-invariant: a `NormalizedUrl` is the sole validation
215
+ chokepoint (it rejects userinfo), and secrets never surface unless intentionally consumed.
216
+
217
+ ## Deliberate non-features
218
+
219
+ * No OCR — scanned or image-only PDFs/pages yield no text.
220
+ * No JS execution — script content is never executed; JS endpoints are surfaced as text or
221
+ links only, never evaluated.
222
+ * No network I/O — extraction is a pure `bytes` → `Extraction` function; DNS resolution and
223
+ host pinning are the caller's responsibility.
224
+ * No DOM for the streaming path — text formats (CSV/JSON/XML) are swept single-pass
225
+ without building a DOM; the HTML/schema path builds a bounded DOM under `max_dom_nodes`.
226
+ * PDF tables are reconstructed from layout rather than a full PDF object model — pdfminer.six
227
+ supplies the tier-1 text sweep; there is no general PDF DOM.
228
+
229
+ ## Guarantees
230
+
231
+ Bounded (every list capped, input size refused), deterministic (same bytes ⇒ same result),
232
+ never raises (hostile input ⇒ `quality="failed"`), and closed (CSS and XPath subsets are
233
+ refused outside, never silently extended).
@@ -0,0 +1,223 @@
1
+ # crawlutron-extract
2
+
3
+ Bounded, deterministic, never-raise content extraction. Converts a byte stream plus a
4
+ content type into a single `Extraction` payload — capped text/markdown, links, frames,
5
+ forms, tables, schema records, a quality grade, and degradation flags — for HTML, office
6
+ containers (DOCX/XLSX/PPTX/EPUB), text formats (CSV/JSON/XML), and a PDF tier-1 text
7
+ sweep via pdfminer.six.
8
+
9
+ There is exactly one entry point, `extract()`. Everything else is derived from its
10
+ `Extraction` result.
11
+
12
+ ## The single entry point
13
+
14
+ ```python
15
+ from crawlutron_extract import extract, DEFAULT_LIMITS
16
+ from crawlutron_extract.models import Extraction
17
+
18
+ result: Extraction = extract(
19
+ content: bytes,
20
+ content_type: str,
21
+ *,
22
+ url: str | None = None,
23
+ limits: ExtractLimits = DEFAULT_LIMITS,
24
+ )
25
+ ```
26
+
27
+ `extract` is the single entry point for every format. It:
28
+
29
+ * never raises — hostile or malformed input yields `quality="failed"` plus a flag,
30
+ * refuses input above `limits.max_input_bytes` (`input_refused`),
31
+ * dispatches by content type to the HTML, office, PDF, or text-format path,
32
+ * returns a fresh, deterministic `Extraction` on every call (two calls on the same bytes
33
+ are byte-identical).
34
+
35
+ Links are admitted through `crawlutron-safe-url` (`NormalizedUrl`), syntax-checked and
36
+ refused when unsafe; DNS-level pinning is the caller's job, not the extractor's.
37
+
38
+ ## Public types
39
+
40
+ All public symbols live in `crawlutron_extract` (see `__all__`).
41
+
42
+ **`Extraction`** — the bounded, deterministic result payload:
43
+ * `text: str` — extracted body text, capped at `max_text_chars`
44
+ * `markdown: str` — markdown rendering, capped at `max_markdown_chars`
45
+ * `charset: str | None` — detected charset for text/HTML
46
+ * `metadata: dict[str, str]` — content type, charset, url, container/format hints
47
+ * `links: list[NormalizedUrl]` — admitted links, capped at `max_links`
48
+ * `frames: list[dict[str, str]]` — frames, capped at `max_frames`
49
+ * `forms: list[dict]` — forms, capped at `max_forms`
50
+ * `tables: list[list[list[str]]]` — tables as row/cell grids, capped at `max_tables`
51
+ * `quality: Quality` — `HIGH` / `MEDIUM` / `LOW` / `FAILED`
52
+ * `flags: list[str]` — degradation signals, never data loss
53
+ * `truncated: bool` — any output list was capped
54
+ * `links_refused: int` — link count refused by the URL policy
55
+ * `records: tuple[Mapping[str, object], ...]` — schema records (schema path only)
56
+
57
+ **`Quality`** (`StrEnum`) — closed ladder: `HIGH`, `MEDIUM`, `LOW`, `FAILED`.
58
+
59
+ **`ExtractLimits`** (frozen dataclass) — every cap applied to an extraction; see
60
+ "Limits and defaults" below. `DEFAULT_LIMITS` is the module-level instance.
61
+
62
+ **Schema types** (`crawlutron_extract.schema`):
63
+ * `FieldSpec(name, selector, kind=FieldKind.TEXT, attribute=None, required=False, many=False)`
64
+ * `FieldKind` — `TEXT`, `MARKDOWN`, `ATTR`, `LINK`, `NUMBER`, `INTEGER`
65
+ * `Schema(record: str | None, fields: tuple[FieldSpec, ...])`
66
+ * `extract_records(html, schema, *, url=None, limits=None) -> Extraction`
67
+
68
+ **Selectors**: `validate_css(selector)`, `validate_xpath(expr)`, both raise
69
+ `SelectorError` when an expression is outside the supported subset.
70
+
71
+ Other public doors: `extract_discovered_links`, `extract_js_endpoints`,
72
+ `extract_sitemap_locs`, `html_diagnostics`, `detect_pagination`, `LinkRecord`.
73
+
74
+ ## Schema language with a worked example
75
+
76
+ The schema path extracts structured records from HTML using a closed CSS-selector
77
+ subset. A `Schema` pairs an optional record selector with a tuple of `FieldSpec`s. When
78
+ `record` is `None` the whole document is one record.
79
+
80
+ ```python
81
+ from crawlutron_extract import DEFAULT_LIMITS, extract_records
82
+ from crawlutron_extract.schema import FieldKind, FieldSpec, Schema
83
+
84
+ schema = Schema(
85
+ record="article.product",
86
+ fields=(
87
+ FieldSpec("title", "h1", FieldKind.TEXT),
88
+ FieldSpec("price", "span.price", FieldKind.NUMBER),
89
+ FieldSpec("link", "a", FieldKind.LINK, many=True),
90
+ FieldSpec("in_stock", ".stock", FieldKind.TEXT),
91
+ ),
92
+ )
93
+ result = extract_records(html_bytes, schema, url="https://example.com/", limits=DEFAULT_LIMITS)
94
+ # result.records -> tuple of dicts like {"title": "...", "price": 12.5, "link": (...), ...}
95
+ ```
96
+
97
+ `FieldKind.LINK` resolves and admits each `href` through `crawlutron-safe-url`; refused
98
+ links are dropped. `NUMBER`/`INTEGER` coerce text; a missing required field sets
99
+ `schema_required_missing`. Output is bounded by `max_records` and `max_fields_per_record`.
100
+
101
+ ## CSS subset (closed — refused outside)
102
+
103
+ `validate_css` accepts only this subset; anything else raises `SelectorError`:
104
+
105
+ * type selector (`div`), universal (`*`)
106
+ * class (`.c`) and id (`#i`)
107
+ * attribute presence (`[a]`) and equality / prefix / suffix / contains
108
+ (`[a='v']`, `[a^='v']`, `[a$='v']`, `[a*='v']`)
109
+ * combinators: descendant (whitespace), child (`>`), adjacent (`+`)
110
+ * pseudo-classes: `:first-child`, `:last-child`, `:nth-child(n)`, `:not(s)`
111
+ * comma-separated selector lists
112
+
113
+ Refused (outside the subset, raises `SelectorError`): namespaces, pseudo-elements (`::`),
114
+ `:hover`/`:nth-of-type`/`:has`/`:is`/`:where` and any pseudo-class not listed, attribute
115
+ operators other than `= ^= $= *=`, and `:not()` containing a non-simple selector. The
116
+ subset is closed by design: if `validate_css` accepts it, the evaluator can match it; if
117
+ the evaluator cannot match it, `validate_css` rejects it.
118
+
119
+ ## XPath subset (closed — refused outside)
120
+
121
+ `validate_xpath` accepts only this subset; anything else raises `SelectorError` naming the
122
+ offending token:
123
+
124
+ * location steps: `/`, `//`, `.//`
125
+ * node test: `*`, a tag name, `text()`, `@attr`
126
+ * predicates: `[n]`, `[last()]`, `[@a]`, `[@a='v']`
127
+ * functions: `contains()`, `concat()`, `normalize-space()`, `starts-with()`
128
+
129
+ Refused (outside the subset): axes (`self::`, `parent::`, `child::`, `descendant::`,
130
+ `attribute::` and any other), any function not listed, any predicate form not listed, and
131
+ any unknown character or token. The subset is closed: `validate_xpath` rejects everything
132
+ the bounded evaluator cannot execute.
133
+
134
+ ## Limits and defaults
135
+
136
+ `ExtractLimits` (frozen dataclass), defaults shown. `max_input_bytes` refuses oversized
137
+ input before any decode; every list output is capped and sets `truncated`.
138
+
139
+ | Limit | Default | Meaning |
140
+ |---|---|---|
141
+ | `max_input_bytes` | 50,000,000 | refuse input larger than this (50 MB) |
142
+ | `max_text_chars` | 100,000 | cap on extracted `text` |
143
+ | `max_markdown_chars` | 200,000 | cap on extracted `markdown` |
144
+ | `max_links` | 1,000 | cap on admitted links |
145
+ | `max_tables` | 50 | cap on tables |
146
+ | `max_table_rows` | 1,000 | rows per table |
147
+ | `max_table_cols` | 100 | columns per table |
148
+ | `max_frames` | 100 | cap on frames |
149
+ | `max_forms` | 100 | cap on forms |
150
+ | `max_dom_nodes` | 50,000 | DOM node cap (HTML/schema path) |
151
+ | `max_dom_depth` | 50 | DOM depth cap |
152
+ | `max_records` | 10,000 | schema records cap |
153
+ | `max_fields_per_record` | 100 | fields per schema record cap |
154
+ | `max_zip_entries` | 10,000 | office container entry cap |
155
+ | `max_zip_ratio` | 1,000 | office container compression-bomb ratio cap |
156
+
157
+ `RICH_TEXT_FLOOR = 200` is the minimum body-text length for `quality="high"` (requires a
158
+ main-content element and a non-lossy decode).
159
+
160
+ ## Quality ladder
161
+
162
+ Closed vocabulary, evaluated by `_resolve_quality`:
163
+
164
+ | Grade | Meaning |
165
+ |---|---|
166
+ | `HIGH` | main-content element found, body text ≥ `RICH_TEXT_FLOOR`, decode not lossy |
167
+ | `MEDIUM` | body text present, but below the floor or a lossy decode |
168
+ | `LOW` | decoded text or structural content (links/forms/tables) present, no body text / lossy |
169
+ | `FAILED` | no extractable content, refused format, or input refused |
170
+
171
+ A clean non-HTML document with decoded text is `LOW` (no main-content concept). An empty
172
+ result is never above `LOW`.
173
+
174
+ ## Flags vocabulary (degradation, not failure)
175
+
176
+ Flags describe degradation, never data loss. `extract` never returns partial data silently:
177
+ every flag is paired with the result it qualifies, and `truncated` is set whenever a cap
178
+ bit. Representative flags:
179
+
180
+ * `charset_lossy` — text had to be lossily re-encoded
181
+ * `links_refused` — one or more links refused by the URL policy
182
+ * `input_refused` — input exceeded `max_input_bytes`
183
+ * `format_refused` — content type not supported
184
+ * `selector_invalid` / `schema_required_missing` — schema selector or required field issue
185
+ * `records_truncated` / `fields_truncated` — a schema cap was hit
186
+ * `schema_dom_cap` — the DOM cap prevented schema extraction
187
+ * `pdf_encoding_partial` — PDF text sweep could not recover the full encoding
188
+ * `table_grid_degraded` / `nested_table_flattened` — a table was normalized, not dropped
189
+
190
+ Text is never discarded because markdown cannot express a shape: nested tables flatten into
191
+ the enclosing cell (pipes escaped, `nested_table_flattened` set), `colspan` repeats content
192
+ and `rowspan` carries it down, and the grid stays rectangular.
193
+
194
+ ## Hard dependency set
195
+
196
+ `crawlutron-extract` ships no extras and is not stdlib-only. The hard dependencies are
197
+ imported plainly — a broken install fails at import, there is no lazy/fallback path:
198
+
199
+ * `crawlutron-safe-url` — `NormalizedUrl` and `HostPolicy`; its own closure is also hard:
200
+ * `regex` — bounded matcher
201
+ * `publicsuffixlist` — public suffix list
202
+ * `pdfminer.six` — PDF tier-1 text sweep
203
+
204
+ Credentials and URLs are object-invariant: a `NormalizedUrl` is the sole validation
205
+ chokepoint (it rejects userinfo), and secrets never surface unless intentionally consumed.
206
+
207
+ ## Deliberate non-features
208
+
209
+ * No OCR — scanned or image-only PDFs/pages yield no text.
210
+ * No JS execution — script content is never executed; JS endpoints are surfaced as text or
211
+ links only, never evaluated.
212
+ * No network I/O — extraction is a pure `bytes` → `Extraction` function; DNS resolution and
213
+ host pinning are the caller's responsibility.
214
+ * No DOM for the streaming path — text formats (CSV/JSON/XML) are swept single-pass
215
+ without building a DOM; the HTML/schema path builds a bounded DOM under `max_dom_nodes`.
216
+ * PDF tables are reconstructed from layout rather than a full PDF object model — pdfminer.six
217
+ supplies the tier-1 text sweep; there is no general PDF DOM.
218
+
219
+ ## Guarantees
220
+
221
+ Bounded (every list capped, input size refused), deterministic (same bytes ⇒ same result),
222
+ never raises (hostile input ⇒ `quality="failed"`), and closed (CSS and XPath subsets are
223
+ refused outside, never silently extended).
@@ -0,0 +1,19 @@
1
+ # Bench results — crawlutron-extract
2
+
3
+ Median-of-7 extraction timings from `packages/extract/bench/bench_extract.py`.
4
+
5
+ Machine: MacBook-Pro.local, Python 3.14.10 on darwin
6
+ Date: 2026-11-18
7
+
8
+ | Case | Input | Median (s) |
9
+ |---|---|---|
10
+ | HTML 100 KiB | ~100 KiB text/html | 0.007148 |
11
+ | HTML 1 MiB | ~1 MiB text/html | 0.074795 |
12
+ | DOCX small | 5 paragraphs | 0.000055 |
13
+ | DOCX large | 200 paragraphs | 0.000473 |
14
+ | PDF real (golden) | simple1.pdf (849 B) | 0.000024 |
15
+ | schema extraction | ~50 KiB HTML, 2 fields | 0.019998 |
16
+
17
+ Coarse timing gate: 1 MiB HTML under 2 s (observed 0.074795 s). This is the
18
+ suite's only timing assertion; timing is inherently flaky, so the bound is
19
+ deliberately coarse and no other timing assertions exist.
@@ -0,0 +1,101 @@
1
+ """Median-of-N benchmarks for crawlutron_extract extraction timings."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import statistics
6
+ import time
7
+ import zipfile
8
+ from io import BytesIO
9
+ from pathlib import Path
10
+
11
+ from crawlutron_extract import extract
12
+ from crawlutron_extract.schema import FieldKind, FieldSpec, Schema, extract_records
13
+
14
+ REPEATS = 7
15
+ DOCX_CT = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
16
+ GOLDEN = Path(__file__).resolve().parent.parent / "tests" / "golden"
17
+
18
+
19
+ def _median_n(fn, n: int = REPEATS) -> float:
20
+ fn() # warmup
21
+ times: list[float] = []
22
+ for _ in range(n):
23
+ start = time.perf_counter()
24
+ fn()
25
+ times.append(time.perf_counter() - start)
26
+ return statistics.median(times)
27
+
28
+
29
+ def _html_page(size_bytes: int) -> str:
30
+ """Deterministic HTML of roughly ``size_bytes`` bytes."""
31
+ parts = ["<html><body><h1>Bench page</h1>"]
32
+ paragraph = "<p>The quick brown fox jumps over the lazy dog. </p>"
33
+ while sum(len(p.encode("utf-8")) for p in parts) < size_bytes:
34
+ parts.append(paragraph)
35
+ parts.append("</body></html>")
36
+ return "".join(parts)
37
+
38
+
39
+ def _build_docx(paragraphs: int) -> bytes:
40
+ """Minimal DOCX with ``paragraphs`` paragraphs (same shape as the test fixtures)."""
41
+ rows = "".join(
42
+ f"<w:p><w:r><w:t>bench paragraph {i}: alpha beta gamma delta epsilon</w:t></w:r></w:p>"
43
+ for i in range(paragraphs)
44
+ )
45
+ document = (
46
+ '<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">'
47
+ "<w:body>" + rows + "</w:body></w:document>"
48
+ )
49
+ buffer = BytesIO()
50
+ with zipfile.ZipFile(buffer, "w") as zf:
51
+ zf.writestr(
52
+ "[Content_Types].xml",
53
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
54
+ '<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"/>',
55
+ )
56
+ zf.writestr("word/document.xml", document)
57
+ return buffer.getvalue()
58
+
59
+
60
+ def _case(name: str, content_type: str, fn) -> tuple[str, str, float]:
61
+ elapsed = _median_n(fn)
62
+ return name, content_type, elapsed
63
+
64
+
65
+ def main() -> None:
66
+ rows = []
67
+ html_100k = _html_page(100 * 1024).encode("utf-8")
68
+ rows.append(_case("HTML 100 KiB", "text/html", lambda: extract(html_100k, "text/html")))
69
+ html_1mib = _html_page(1024 * 1024).encode("utf-8")
70
+ rows.append(_case("HTML 1 MiB", "text/html", lambda: extract(html_1mib, "text/html")))
71
+ rows.append(_case("DOCX small", DOCX_CT, lambda: extract(_build_docx(5), DOCX_CT)))
72
+ rows.append(_case("DOCX large", DOCX_CT, lambda: extract(_build_docx(200), DOCX_CT)))
73
+ pdf_bytes = (GOLDEN / "simple1.pdf").read_bytes()
74
+ rows.append(_case("PDF real (golden)", "application/pdf", lambda: extract(pdf_bytes, "application/pdf")))
75
+ schema = Schema(
76
+ record="article",
77
+ fields=(
78
+ FieldSpec("title", "h1", FieldKind.TEXT),
79
+ FieldSpec("body", "p", FieldKind.TEXT, many=True),
80
+ ),
81
+ )
82
+ rows.append(
83
+ _case(
84
+ "schema extraction",
85
+ "text/html",
86
+ lambda: extract_records(_html_page(50 * 1024), schema),
87
+ )
88
+ )
89
+
90
+ print("crawlutron-extract bench (median of", REPEATS, "repeats)")
91
+ print()
92
+ print(f"{'Case':<24} {'Content type':<48} {'median (s)'}")
93
+ print("-" * 78)
94
+ for name, ctype, elapsed in rows:
95
+ print(f"{name:<24} {ctype:<48} {elapsed:.6f}")
96
+ print()
97
+ print("Slowest:", max(rows, key=lambda r: r[2])[0])
98
+
99
+
100
+ if __name__ == "__main__":
101
+ main()
@@ -0,0 +1,20 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "crawlutron-extract"
7
+ version = "0.5.0"
8
+ description = "Content extraction: HTML, office containers, text formats, PDF tier 1, bounded and deterministic"
9
+ readme = "README.md"
10
+ requires-python = ">=3.14"
11
+ license = "MIT"
12
+
13
+ # crawlutron-safe-url is required for link admission (slice 5). Its own
14
+ # dependency closure includes regex (bounded matcher) and publicsuffixlist
15
+ # (public suffix list) as hard dependencies; the default install is not
16
+ # stdlib-only. pdfminer.six is a hard dependency for PDF tier-1 fallback.
17
+ dependencies = ["crawlutron-safe-url", "pdfminer.six==20260107"]
18
+
19
+ [tool.hatch.build.targets.wheel]
20
+ packages = ["src/crawlutron_extract"]
@@ -0,0 +1,36 @@
1
+ """crawlutron_extract: stdlib-only content extraction, bounded and deterministic."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from crawlutron_extract._extract import DEFAULT_LIMITS, extract
6
+ from crawlutron_extract.css_select import SelectorError, validate_css
7
+ from crawlutron_extract.html_diagnostics import html_diagnostics
8
+ from crawlutron_extract.js_endpoints import extract_js_endpoints
9
+ from crawlutron_extract.limits import ExtractLimits
10
+ from crawlutron_extract.links import LinkRecord, extract_discovered_links
11
+ from crawlutron_extract.models import Extraction, Quality
12
+ from crawlutron_extract.pagination import detect_pagination
13
+ from crawlutron_extract.schema import FieldKind, FieldSpec, Schema, extract_records
14
+ from crawlutron_extract.sitemap import extract_sitemap_locs
15
+ from crawlutron_extract.xpath_select import validate_xpath
16
+
17
+ __all__ = [
18
+ "DEFAULT_LIMITS",
19
+ "ExtractLimits",
20
+ "Extraction",
21
+ "FieldKind",
22
+ "FieldSpec",
23
+ "Quality",
24
+ "Schema",
25
+ "extract",
26
+ "extract_discovered_links",
27
+ "extract_js_endpoints",
28
+ "extract_records",
29
+ "extract_sitemap_locs",
30
+ "html_diagnostics",
31
+ "LinkRecord",
32
+ "SelectorError",
33
+ "detect_pagination",
34
+ "validate_css",
35
+ "validate_xpath",
36
+ ]