crawlutron-extract 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- crawlutron_extract-0.5.0/.gitignore +38 -0
- crawlutron_extract-0.5.0/PKG-INFO +233 -0
- crawlutron_extract-0.5.0/README.md +223 -0
- crawlutron_extract-0.5.0/bench/RESULTS.md +19 -0
- crawlutron_extract-0.5.0/bench/bench_extract.py +101 -0
- crawlutron_extract-0.5.0/pyproject.toml +20 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/__init__.py +36 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/_extract.py +338 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/_html_common.py +49 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/_select_common.py +84 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/charset.py +94 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/css_select.py +447 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/html_diagnostics.py +72 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/html_dom.py +114 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/html_forms_frames.py +109 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/html_links.py +84 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/html_meta.py +160 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/html_pass.py +212 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/html_tables.py +213 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/html_text.py +211 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/js_endpoints.py +55 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/limits.py +59 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/links.py +109 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/models.py +39 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/office.py +360 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/pagination.py +50 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/pdf_pass.py +351 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/py.typed +0 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/schema.py +248 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/sitemap.py +59 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/text_format.py +281 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/xpath_eval.py +143 -0
- crawlutron_extract-0.5.0/src/crawlutron_extract/xpath_select.py +283 -0
- crawlutron_extract-0.5.0/tests/conftest.py +139 -0
- crawlutron_extract-0.5.0/tests/fixtures/degraded_table.pdf +39 -0
- crawlutron_extract-0.5.0/tests/fixtures/generate_xpath_golden.py +97 -0
- crawlutron_extract-0.5.0/tests/fixtures/jo.pdf +0 -0
- crawlutron_extract-0.5.0/tests/fixtures/sample_table.pdf +39 -0
- crawlutron_extract-0.5.0/tests/fixtures/simple2.pdf +78 -0
- crawlutron_extract-0.5.0/tests/fixtures/xpath_golden.json +148 -0
- crawlutron_extract-0.5.0/tests/fixtures/zen_of_python_corrupted.pdf +0 -0
- crawlutron_extract-0.5.0/tests/golden/123233_charts.xlsx +0 -0
- crawlutron_extract-0.5.0/tests/golden/1_NoIden.xlsx +0 -0
- crawlutron_extract-0.5.0/tests/golden/49386-null_dates.pptx +0 -0
- crawlutron_extract-0.5.0/tests/golden/52288.docx +0 -0
- crawlutron_extract-0.5.0/tests/golden/55733.docx +0 -0
- crawlutron_extract-0.5.0/tests/golden/LICENSES.md +49 -0
- crawlutron_extract-0.5.0/tests/golden/encoding_test-yahoo-jp.dat +10 -0
- crawlutron_extract-0.5.0/tests/golden/encoding_tests2.dat +115 -0
- crawlutron_extract-0.5.0/tests/golden/sampleOneByteIdentityEncode.pdf +0 -0
- crawlutron_extract-0.5.0/tests/golden/simple1.pdf +68 -0
- crawlutron_extract-0.5.0/tests/golden/simple3.pdf +87 -0
- crawlutron_extract-0.5.0/tests/golden/test_pattern_colors.pdf +55 -0
- crawlutron_extract-0.5.0/tests/golden/tokenizer_escapeFlag.test +36 -0
- crawlutron_extract-0.5.0/tests/golden/tokenizer_test1.test +353 -0
- crawlutron_extract-0.5.0/tests/golden/tokenizer_unicodeCharsProblematic.test +41 -0
- crawlutron_extract-0.5.0/tests/golden/tokenizer_xmlViolation.test +20 -0
- crawlutron_extract-0.5.0/tests/test_bench_timing.py +25 -0
- crawlutron_extract-0.5.0/tests/test_charset.py +167 -0
- crawlutron_extract-0.5.0/tests/test_css_select.py +53 -0
- crawlutron_extract-0.5.0/tests/test_detect_pagination.py +40 -0
- crawlutron_extract-0.5.0/tests/test_extract_discovered_links.py +87 -0
- crawlutron_extract-0.5.0/tests/test_extract_js_endpoints.py +48 -0
- crawlutron_extract-0.5.0/tests/test_extract_sitemap_locs.py +74 -0
- crawlutron_extract-0.5.0/tests/test_golden_corpus.py +178 -0
- crawlutron_extract-0.5.0/tests/test_html_diagnostics.py +62 -0
- crawlutron_extract-0.5.0/tests/test_html_forms_frames.py +120 -0
- crawlutron_extract-0.5.0/tests/test_html_metadata.py +141 -0
- crawlutron_extract-0.5.0/tests/test_html_pass.py +499 -0
- crawlutron_extract-0.5.0/tests/test_office.py +499 -0
- crawlutron_extract-0.5.0/tests/test_pdf_pass.py +123 -0
- crawlutron_extract-0.5.0/tests/test_pdf_tier2.py +67 -0
- crawlutron_extract-0.5.0/tests/test_schema.py +298 -0
- crawlutron_extract-0.5.0/tests/test_skeleton.py +126 -0
- crawlutron_extract-0.5.0/tests/test_text_format.py +153 -0
- crawlutron_extract-0.5.0/tests/test_xpath_adversarial.py +92 -0
- crawlutron_extract-0.5.0/tests/test_xpath_golden.py +58 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.py[cod]
|
|
3
|
+
.pytest_cache/
|
|
4
|
+
report/
|
|
5
|
+
# Build artifacts from `python -m build` / editable installs (#26).
|
|
6
|
+
dist/
|
|
7
|
+
build/
|
|
8
|
+
# Both spellings: a linked worktree reaches the venv through a `.venv` symlink,
|
|
9
|
+
# which the trailing-slash form does not match -- and a committed symlink into
|
|
10
|
+
# one checkout's venv is worse than no venv at all.
|
|
11
|
+
.venv/
|
|
12
|
+
.venv
|
|
13
|
+
.dynamic-task-ledger.json
|
|
14
|
+
*.egg-info/
|
|
15
|
+
.cache/
|
|
16
|
+
.mypy_cache/
|
|
17
|
+
.ruff_cache/
|
|
18
|
+
.gate_fanout/
|
|
19
|
+
.coverage
|
|
20
|
+
htmlcov/
|
|
21
|
+
# Session-analysis residue written by the cortexkit tooling. It is ephemeral
|
|
22
|
+
# per-session output, not project content, and its own managed block already
|
|
23
|
+
# declares `magic-context/` untracked -- the historian XML under it was
|
|
24
|
+
# committed anyway, which is how a directory full of transcript digests ended
|
|
25
|
+
# up in the tree this service is extracted from.
|
|
26
|
+
.cortexkit/
|
|
27
|
+
logs/
|
|
28
|
+
cache/
|
|
29
|
+
.env
|
|
30
|
+
tmp/
|
|
31
|
+
*.log
|
|
32
|
+
.DS_Store
|
|
33
|
+
|
|
34
|
+
.coverage.*
|
|
35
|
+
.opencode
|
|
36
|
+
.hardstand/
|
|
37
|
+
|
|
38
|
+
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: crawlutron-extract
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: Content extraction: HTML, office containers, text formats, PDF tier 1, bounded and deterministic
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Requires-Python: >=3.14
|
|
7
|
+
Requires-Dist: crawlutron-safe-url
|
|
8
|
+
Requires-Dist: pdfminer-six==20260107
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
|
|
11
|
+
# crawlutron-extract
|
|
12
|
+
|
|
13
|
+
Bounded, deterministic, never-raise content extraction. Converts a byte stream plus a
|
|
14
|
+
content type into a single `Extraction` payload — capped text/markdown, links, frames,
|
|
15
|
+
forms, tables, schema records, a quality grade, and degradation flags — for HTML, office
|
|
16
|
+
containers (DOCX/XLSX/PPTX/EPUB), text formats (CSV/JSON/XML), and a PDF tier-1 text
|
|
17
|
+
sweep via pdfminer.six.
|
|
18
|
+
|
|
19
|
+
There is exactly one entry point, `extract()`. Everything else is derived from its
|
|
20
|
+
`Extraction` result.
|
|
21
|
+
|
|
22
|
+
## The single entry point
|
|
23
|
+
|
|
24
|
+
```python
|
|
25
|
+
from crawlutron_extract import extract, DEFAULT_LIMITS
|
|
26
|
+
from crawlutron_extract.models import Extraction
|
|
27
|
+
|
|
28
|
+
result: Extraction = extract(
|
|
29
|
+
content: bytes,
|
|
30
|
+
content_type: str,
|
|
31
|
+
*,
|
|
32
|
+
url: str | None = None,
|
|
33
|
+
limits: ExtractLimits = DEFAULT_LIMITS,
|
|
34
|
+
)
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
`extract` is the single entry point for every format. It:
|
|
38
|
+
|
|
39
|
+
* never raises — hostile or malformed input yields `quality="failed"` plus a flag,
|
|
40
|
+
* refuses input above `limits.max_input_bytes` (`input_refused`),
|
|
41
|
+
* dispatches by content type to the HTML, office, PDF, or text-format path,
|
|
42
|
+
* returns a fresh, deterministic `Extraction` on every call (two calls on the same bytes
|
|
43
|
+
are byte-identical).
|
|
44
|
+
|
|
45
|
+
Links are admitted through `crawlutron-safe-url` (`NormalizedUrl`), syntax-checked and
|
|
46
|
+
refused when unsafe; DNS-level pinning is the caller's job, not the extractor's.
|
|
47
|
+
|
|
48
|
+
## Public types
|
|
49
|
+
|
|
50
|
+
All public symbols live in `crawlutron_extract` (see `__all__`).
|
|
51
|
+
|
|
52
|
+
**`Extraction`** — the bounded, deterministic result payload:
|
|
53
|
+
* `text: str` — extracted body text, capped at `max_text_chars`
|
|
54
|
+
* `markdown: str` — markdown rendering, capped at `max_markdown_chars`
|
|
55
|
+
* `charset: str | None` — detected charset for text/HTML
|
|
56
|
+
* `metadata: dict[str, str]` — content type, charset, url, container/format hints
|
|
57
|
+
* `links: list[NormalizedUrl]` — admitted links, capped at `max_links`
|
|
58
|
+
* `frames: list[dict[str, str]]` — frames, capped at `max_frames`
|
|
59
|
+
* `forms: list[dict]` — forms, capped at `max_forms`
|
|
60
|
+
* `tables: list[list[list[str]]]` — tables as row/cell grids, capped at `max_tables`
|
|
61
|
+
* `quality: Quality` — `HIGH` / `MEDIUM` / `LOW` / `FAILED`
|
|
62
|
+
* `flags: list[str]` — degradation signals, never data loss
|
|
63
|
+
* `truncated: bool` — any output list was capped
|
|
64
|
+
* `links_refused: int` — link count refused by the URL policy
|
|
65
|
+
* `records: tuple[Mapping[str, object], ...]` — schema records (schema path only)
|
|
66
|
+
|
|
67
|
+
**`Quality`** (`StrEnum`) — closed ladder: `HIGH`, `MEDIUM`, `LOW`, `FAILED`.
|
|
68
|
+
|
|
69
|
+
**`ExtractLimits`** (frozen dataclass) — every cap applied to an extraction; see
|
|
70
|
+
"Limits and defaults" below. `DEFAULT_LIMITS` is the module-level instance.
|
|
71
|
+
|
|
72
|
+
**Schema types** (`crawlutron_extract.schema`):
|
|
73
|
+
* `FieldSpec(name, selector, kind=FieldKind.TEXT, attribute=None, required=False, many=False)`
|
|
74
|
+
* `FieldKind` — `TEXT`, `MARKDOWN`, `ATTR`, `LINK`, `NUMBER`, `INTEGER`
|
|
75
|
+
* `Schema(record: str | None, fields: tuple[FieldSpec, ...])`
|
|
76
|
+
* `extract_records(html, schema, *, url=None, limits=None) -> Extraction`
|
|
77
|
+
|
|
78
|
+
**Selectors**: `validate_css(selector)`, `validate_xpath(expr)`, both raise
|
|
79
|
+
`SelectorError` when an expression is outside the supported subset.
|
|
80
|
+
|
|
81
|
+
Other public doors: `extract_discovered_links`, `extract_js_endpoints`,
|
|
82
|
+
`extract_sitemap_locs`, `html_diagnostics`, `detect_pagination`, `LinkRecord`.
|
|
83
|
+
|
|
84
|
+
## Schema language with a worked example
|
|
85
|
+
|
|
86
|
+
The schema path extracts structured records from HTML using a closed CSS-selector
|
|
87
|
+
subset. A `Schema` pairs an optional record selector with a tuple of `FieldSpec`s. When
|
|
88
|
+
`record` is `None` the whole document is one record.
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from crawlutron_extract import DEFAULT_LIMITS, extract_records
|
|
92
|
+
from crawlutron_extract.schema import FieldKind, FieldSpec, Schema
|
|
93
|
+
|
|
94
|
+
schema = Schema(
|
|
95
|
+
record="article.product",
|
|
96
|
+
fields=(
|
|
97
|
+
FieldSpec("title", "h1", FieldKind.TEXT),
|
|
98
|
+
FieldSpec("price", "span.price", FieldKind.NUMBER),
|
|
99
|
+
FieldSpec("link", "a", FieldKind.LINK, many=True),
|
|
100
|
+
FieldSpec("in_stock", ".stock", FieldKind.TEXT),
|
|
101
|
+
),
|
|
102
|
+
)
|
|
103
|
+
result = extract_records(html_bytes, schema, url="https://example.com/", limits=DEFAULT_LIMITS)
|
|
104
|
+
# result.records -> tuple of dicts like {"title": "...", "price": 12.5, "link": (...), ...}
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
`FieldKind.LINK` resolves and admits each `href` through `crawlutron-safe-url`; refused
|
|
108
|
+
links are dropped. `NUMBER`/`INTEGER` coerce text; a missing required field sets
|
|
109
|
+
`schema_required_missing`. Output is bounded by `max_records` and `max_fields_per_record`.
|
|
110
|
+
|
|
111
|
+
## CSS subset (closed — refused outside)
|
|
112
|
+
|
|
113
|
+
`validate_css` accepts only this subset; anything else raises `SelectorError`:
|
|
114
|
+
|
|
115
|
+
* type selector (`div`), universal (`*`)
|
|
116
|
+
* class (`.c`) and id (`#i`)
|
|
117
|
+
* attribute presence (`[a]`) and equality / prefix / suffix / contains
|
|
118
|
+
(`[a='v']`, `[a^='v']`, `[a$='v']`, `[a*='v']`)
|
|
119
|
+
* combinators: descendant (whitespace), child (`>`), adjacent (`+`)
|
|
120
|
+
* pseudo-classes: `:first-child`, `:last-child`, `:nth-child(n)`, `:not(s)`
|
|
121
|
+
* comma-separated selector lists
|
|
122
|
+
|
|
123
|
+
Refused (outside the subset, raises `SelectorError`): namespaces, pseudo-elements (`::`),
|
|
124
|
+
`:hover`/`:nth-of-type`/`:has`/`:is`/`:where` and any pseudo-class not listed, attribute
|
|
125
|
+
operators other than `= ^= $= *=`, and `:not()` containing a non-simple selector. The
|
|
126
|
+
subset is closed by design: if `validate_css` accepts it, the evaluator can match it; if
|
|
127
|
+
the evaluator cannot match it, `validate_css` rejects it.
|
|
128
|
+
|
|
129
|
+
## XPath subset (closed — refused outside)
|
|
130
|
+
|
|
131
|
+
`validate_xpath` accepts only this subset; anything else raises `SelectorError` naming the
|
|
132
|
+
offending token:
|
|
133
|
+
|
|
134
|
+
* location steps: `/`, `//`, `.//`
|
|
135
|
+
* node test: `*`, a tag name, `text()`, `@attr`
|
|
136
|
+
* predicates: `[n]`, `[last()]`, `[@a]`, `[@a='v']`
|
|
137
|
+
* functions: `contains()`, `concat()`, `normalize-space()`, `starts-with()`
|
|
138
|
+
|
|
139
|
+
Refused (outside the subset): axes (`self::`, `parent::`, `child::`, `descendant::`,
|
|
140
|
+
`attribute::` and any other), any function not listed, any predicate form not listed, and
|
|
141
|
+
any unknown character or token. The subset is closed: `validate_xpath` rejects everything
|
|
142
|
+
the bounded evaluator cannot execute.
|
|
143
|
+
|
|
144
|
+
## Limits and defaults
|
|
145
|
+
|
|
146
|
+
`ExtractLimits` (frozen dataclass), defaults shown. `max_input_bytes` refuses oversized
|
|
147
|
+
input before any decode; every list output is capped and sets `truncated`.
|
|
148
|
+
|
|
149
|
+
| Limit | Default | Meaning |
|
|
150
|
+
|---|---|---|
|
|
151
|
+
| `max_input_bytes` | 50,000,000 | refuse input larger than this (50 MB) |
|
|
152
|
+
| `max_text_chars` | 100,000 | cap on extracted `text` |
|
|
153
|
+
| `max_markdown_chars` | 200,000 | cap on extracted `markdown` |
|
|
154
|
+
| `max_links` | 1,000 | cap on admitted links |
|
|
155
|
+
| `max_tables` | 50 | cap on tables |
|
|
156
|
+
| `max_table_rows` | 1,000 | rows per table |
|
|
157
|
+
| `max_table_cols` | 100 | columns per table |
|
|
158
|
+
| `max_frames` | 100 | cap on frames |
|
|
159
|
+
| `max_forms` | 100 | cap on forms |
|
|
160
|
+
| `max_dom_nodes` | 50,000 | DOM node cap (HTML/schema path) |
|
|
161
|
+
| `max_dom_depth` | 50 | DOM depth cap |
|
|
162
|
+
| `max_records` | 10,000 | schema records cap |
|
|
163
|
+
| `max_fields_per_record` | 100 | fields per schema record cap |
|
|
164
|
+
| `max_zip_entries` | 10,000 | office container entry cap |
|
|
165
|
+
| `max_zip_ratio` | 1,000 | office container compression-bomb ratio cap |
|
|
166
|
+
|
|
167
|
+
`RICH_TEXT_FLOOR = 200` is the minimum body-text length for `quality="high"` (requires a
|
|
168
|
+
main-content element and a non-lossy decode).
|
|
169
|
+
|
|
170
|
+
## Quality ladder
|
|
171
|
+
|
|
172
|
+
Closed vocabulary, evaluated by `_resolve_quality`:
|
|
173
|
+
|
|
174
|
+
| Grade | Meaning |
|
|
175
|
+
|---|---|
|
|
176
|
+
| `HIGH` | main-content element found, body text ≥ `RICH_TEXT_FLOOR`, decode not lossy |
|
|
177
|
+
| `MEDIUM` | body text present, but below the floor or a lossy decode |
|
|
178
|
+
| `LOW` | decoded text or structural content (links/forms/tables) present, no body text / lossy |
|
|
179
|
+
| `FAILED` | no extractable content, refused format, or input refused |
|
|
180
|
+
|
|
181
|
+
A clean non-HTML document with decoded text is `LOW` (no main-content concept). An empty
|
|
182
|
+
result is never above `LOW`.
|
|
183
|
+
|
|
184
|
+
## Flags vocabulary (degradation, not failure)
|
|
185
|
+
|
|
186
|
+
Flags describe degradation, never data loss. `extract` never returns partial data silently:
|
|
187
|
+
every flag is paired with the result it qualifies, and `truncated` is set whenever a cap
|
|
188
|
+
bit. Representative flags:
|
|
189
|
+
|
|
190
|
+
* `charset_lossy` — text had to be lossily re-encoded
|
|
191
|
+
* `links_refused` — one or more links refused by the URL policy
|
|
192
|
+
* `input_refused` — input exceeded `max_input_bytes`
|
|
193
|
+
* `format_refused` — content type not supported
|
|
194
|
+
* `selector_invalid` / `schema_required_missing` — schema selector or required field issue
|
|
195
|
+
* `records_truncated` / `fields_truncated` — a schema cap was hit
|
|
196
|
+
* `schema_dom_cap` — the DOM cap prevented schema extraction
|
|
197
|
+
* `pdf_encoding_partial` — PDF text sweep could not recover the full encoding
|
|
198
|
+
* `table_grid_degraded` / `nested_table_flattened` — a table was normalized, not dropped
|
|
199
|
+
|
|
200
|
+
Text is never discarded because markdown cannot express a shape: nested tables flatten into
|
|
201
|
+
the enclosing cell (pipes escaped, `nested_table_flattened` set), `colspan` repeats content
|
|
202
|
+
and `rowspan` carries it down, and the grid stays rectangular.
|
|
203
|
+
|
|
204
|
+
## Hard dependency set
|
|
205
|
+
|
|
206
|
+
`crawlutron-extract` ships no extras and is not stdlib-only. The hard dependencies are
|
|
207
|
+
imported plainly — a broken install fails at import, there is no lazy/fallback path:
|
|
208
|
+
|
|
209
|
+
* `crawlutron-safe-url` — `NormalizedUrl` and `HostPolicy`; its own closure is also hard:
|
|
210
|
+
* `regex` — bounded matcher
|
|
211
|
+
* `publicsuffixlist` — public suffix list
|
|
212
|
+
* `pdfminer.six` — PDF tier-1 text sweep
|
|
213
|
+
|
|
214
|
+
Credentials and URLs are object-invariant: a `NormalizedUrl` is the sole validation
|
|
215
|
+
chokepoint (it rejects userinfo), and secrets never surface unless intentionally consumed.
|
|
216
|
+
|
|
217
|
+
## Deliberate non-features
|
|
218
|
+
|
|
219
|
+
* No OCR — scanned or image-only PDFs/pages yield no text.
|
|
220
|
+
* No JS execution — script content is never executed; JS endpoints are surfaced as text or
|
|
221
|
+
links only, never evaluated.
|
|
222
|
+
* No network I/O — extraction is a pure `bytes` → `Extraction` function; DNS resolution and
|
|
223
|
+
host pinning are the caller's responsibility.
|
|
224
|
+
* No DOM for the streaming path — text formats (CSV/JSON/XML) are swept single-pass
|
|
225
|
+
without building a DOM; the HTML/schema path builds a bounded DOM under `max_dom_nodes`.
|
|
226
|
+
* PDF tables are reconstructed from layout rather than a full PDF object model — pdfminer.six
|
|
227
|
+
supplies the tier-1 text sweep; there is no general PDF DOM.
|
|
228
|
+
|
|
229
|
+
## Guarantees
|
|
230
|
+
|
|
231
|
+
Bounded (every list capped, input size refused), deterministic (same bytes ⇒ same result),
|
|
232
|
+
never raises (hostile input ⇒ `quality="failed"`), and closed (CSS and XPath subsets are
|
|
233
|
+
refused outside, never silently extended).
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
# crawlutron-extract
|
|
2
|
+
|
|
3
|
+
Bounded, deterministic, never-raise content extraction. Converts a byte stream plus a
|
|
4
|
+
content type into a single `Extraction` payload — capped text/markdown, links, frames,
|
|
5
|
+
forms, tables, schema records, a quality grade, and degradation flags — for HTML, office
|
|
6
|
+
containers (DOCX/XLSX/PPTX/EPUB), text formats (CSV/JSON/XML), and a PDF tier-1 text
|
|
7
|
+
sweep via pdfminer.six.
|
|
8
|
+
|
|
9
|
+
There is exactly one entry point, `extract()`. Everything else is derived from its
|
|
10
|
+
`Extraction` result.
|
|
11
|
+
|
|
12
|
+
## The single entry point
|
|
13
|
+
|
|
14
|
+
```python
|
|
15
|
+
from crawlutron_extract import extract, DEFAULT_LIMITS
|
|
16
|
+
from crawlutron_extract.models import Extraction
|
|
17
|
+
|
|
18
|
+
result: Extraction = extract(
|
|
19
|
+
content: bytes,
|
|
20
|
+
content_type: str,
|
|
21
|
+
*,
|
|
22
|
+
url: str | None = None,
|
|
23
|
+
limits: ExtractLimits = DEFAULT_LIMITS,
|
|
24
|
+
)
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
`extract` is the single entry point for every format. It:
|
|
28
|
+
|
|
29
|
+
* never raises — hostile or malformed input yields `quality="failed"` plus a flag,
|
|
30
|
+
* refuses input above `limits.max_input_bytes` (`input_refused`),
|
|
31
|
+
* dispatches by content type to the HTML, office, PDF, or text-format path,
|
|
32
|
+
* returns a fresh, deterministic `Extraction` on every call (two calls on the same bytes
|
|
33
|
+
are byte-identical).
|
|
34
|
+
|
|
35
|
+
Links are admitted through `crawlutron-safe-url` (`NormalizedUrl`), syntax-checked and
|
|
36
|
+
refused when unsafe; DNS-level pinning is the caller's job, not the extractor's.
|
|
37
|
+
|
|
38
|
+
## Public types
|
|
39
|
+
|
|
40
|
+
All public symbols live in `crawlutron_extract` (see `__all__`).
|
|
41
|
+
|
|
42
|
+
**`Extraction`** — the bounded, deterministic result payload:
|
|
43
|
+
* `text: str` — extracted body text, capped at `max_text_chars`
|
|
44
|
+
* `markdown: str` — markdown rendering, capped at `max_markdown_chars`
|
|
45
|
+
* `charset: str | None` — detected charset for text/HTML
|
|
46
|
+
* `metadata: dict[str, str]` — content type, charset, url, container/format hints
|
|
47
|
+
* `links: list[NormalizedUrl]` — admitted links, capped at `max_links`
|
|
48
|
+
* `frames: list[dict[str, str]]` — frames, capped at `max_frames`
|
|
49
|
+
* `forms: list[dict]` — forms, capped at `max_forms`
|
|
50
|
+
* `tables: list[list[list[str]]]` — tables as row/cell grids, capped at `max_tables`
|
|
51
|
+
* `quality: Quality` — `HIGH` / `MEDIUM` / `LOW` / `FAILED`
|
|
52
|
+
* `flags: list[str]` — degradation signals, never data loss
|
|
53
|
+
* `truncated: bool` — any output list was capped
|
|
54
|
+
* `links_refused: int` — link count refused by the URL policy
|
|
55
|
+
* `records: tuple[Mapping[str, object], ...]` — schema records (schema path only)
|
|
56
|
+
|
|
57
|
+
**`Quality`** (`StrEnum`) — closed ladder: `HIGH`, `MEDIUM`, `LOW`, `FAILED`.
|
|
58
|
+
|
|
59
|
+
**`ExtractLimits`** (frozen dataclass) — every cap applied to an extraction; see
|
|
60
|
+
"Limits and defaults" below. `DEFAULT_LIMITS` is the module-level instance.
|
|
61
|
+
|
|
62
|
+
**Schema types** (`crawlutron_extract.schema`):
|
|
63
|
+
* `FieldSpec(name, selector, kind=FieldKind.TEXT, attribute=None, required=False, many=False)`
|
|
64
|
+
* `FieldKind` — `TEXT`, `MARKDOWN`, `ATTR`, `LINK`, `NUMBER`, `INTEGER`
|
|
65
|
+
* `Schema(record: str | None, fields: tuple[FieldSpec, ...])`
|
|
66
|
+
* `extract_records(html, schema, *, url=None, limits=None) -> Extraction`
|
|
67
|
+
|
|
68
|
+
**Selectors**: `validate_css(selector)`, `validate_xpath(expr)`, both raise
|
|
69
|
+
`SelectorError` when an expression is outside the supported subset.
|
|
70
|
+
|
|
71
|
+
Other public doors: `extract_discovered_links`, `extract_js_endpoints`,
|
|
72
|
+
`extract_sitemap_locs`, `html_diagnostics`, `detect_pagination`, `LinkRecord`.
|
|
73
|
+
|
|
74
|
+
## Schema language with a worked example
|
|
75
|
+
|
|
76
|
+
The schema path extracts structured records from HTML using a closed CSS-selector
|
|
77
|
+
subset. A `Schema` pairs an optional record selector with a tuple of `FieldSpec`s. When
|
|
78
|
+
`record` is `None` the whole document is one record.
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
from crawlutron_extract import DEFAULT_LIMITS, extract_records
|
|
82
|
+
from crawlutron_extract.schema import FieldKind, FieldSpec, Schema
|
|
83
|
+
|
|
84
|
+
schema = Schema(
|
|
85
|
+
record="article.product",
|
|
86
|
+
fields=(
|
|
87
|
+
FieldSpec("title", "h1", FieldKind.TEXT),
|
|
88
|
+
FieldSpec("price", "span.price", FieldKind.NUMBER),
|
|
89
|
+
FieldSpec("link", "a", FieldKind.LINK, many=True),
|
|
90
|
+
FieldSpec("in_stock", ".stock", FieldKind.TEXT),
|
|
91
|
+
),
|
|
92
|
+
)
|
|
93
|
+
result = extract_records(html_bytes, schema, url="https://example.com/", limits=DEFAULT_LIMITS)
|
|
94
|
+
# result.records -> tuple of dicts like {"title": "...", "price": 12.5, "link": (...), ...}
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
`FieldKind.LINK` resolves and admits each `href` through `crawlutron-safe-url`; refused
|
|
98
|
+
links are dropped. `NUMBER`/`INTEGER` coerce text; a missing required field sets
|
|
99
|
+
`schema_required_missing`. Output is bounded by `max_records` and `max_fields_per_record`.
|
|
100
|
+
|
|
101
|
+
## CSS subset (closed — refused outside)
|
|
102
|
+
|
|
103
|
+
`validate_css` accepts only this subset; anything else raises `SelectorError`:
|
|
104
|
+
|
|
105
|
+
* type selector (`div`), universal (`*`)
|
|
106
|
+
* class (`.c`) and id (`#i`)
|
|
107
|
+
* attribute presence (`[a]`) and equality / prefix / suffix / contains
|
|
108
|
+
(`[a='v']`, `[a^='v']`, `[a$='v']`, `[a*='v']`)
|
|
109
|
+
* combinators: descendant (whitespace), child (`>`), adjacent (`+`)
|
|
110
|
+
* pseudo-classes: `:first-child`, `:last-child`, `:nth-child(n)`, `:not(s)`
|
|
111
|
+
* comma-separated selector lists
|
|
112
|
+
|
|
113
|
+
Refused (outside the subset, raises `SelectorError`): namespaces, pseudo-elements (`::`),
|
|
114
|
+
`:hover`/`:nth-of-type`/`:has`/`:is`/`:where` and any pseudo-class not listed, attribute
|
|
115
|
+
operators other than `= ^= $= *=`, and `:not()` containing a non-simple selector. The
|
|
116
|
+
subset is closed by design: if `validate_css` accepts it, the evaluator can match it; if
|
|
117
|
+
the evaluator cannot match it, `validate_css` rejects it.
|
|
118
|
+
|
|
119
|
+
## XPath subset (closed — refused outside)
|
|
120
|
+
|
|
121
|
+
`validate_xpath` accepts only this subset; anything else raises `SelectorError` naming the
|
|
122
|
+
offending token:
|
|
123
|
+
|
|
124
|
+
* location steps: `/`, `//`, `.//`
|
|
125
|
+
* node test: `*`, a tag name, `text()`, `@attr`
|
|
126
|
+
* predicates: `[n]`, `[last()]`, `[@a]`, `[@a='v']`
|
|
127
|
+
* functions: `contains()`, `concat()`, `normalize-space()`, `starts-with()`
|
|
128
|
+
|
|
129
|
+
Refused (outside the subset): axes (`self::`, `parent::`, `child::`, `descendant::`,
|
|
130
|
+
`attribute::` and any other), any function not listed, any predicate form not listed, and
|
|
131
|
+
any unknown character or token. The subset is closed: `validate_xpath` rejects everything
|
|
132
|
+
the bounded evaluator cannot execute.
|
|
133
|
+
|
|
134
|
+
## Limits and defaults
|
|
135
|
+
|
|
136
|
+
`ExtractLimits` (frozen dataclass), defaults shown. `max_input_bytes` refuses oversized
|
|
137
|
+
input before any decode; every list output is capped and sets `truncated`.
|
|
138
|
+
|
|
139
|
+
| Limit | Default | Meaning |
|
|
140
|
+
|---|---|---|
|
|
141
|
+
| `max_input_bytes` | 50,000,000 | refuse input larger than this (50 MB) |
|
|
142
|
+
| `max_text_chars` | 100,000 | cap on extracted `text` |
|
|
143
|
+
| `max_markdown_chars` | 200,000 | cap on extracted `markdown` |
|
|
144
|
+
| `max_links` | 1,000 | cap on admitted links |
|
|
145
|
+
| `max_tables` | 50 | cap on tables |
|
|
146
|
+
| `max_table_rows` | 1,000 | rows per table |
|
|
147
|
+
| `max_table_cols` | 100 | columns per table |
|
|
148
|
+
| `max_frames` | 100 | cap on frames |
|
|
149
|
+
| `max_forms` | 100 | cap on forms |
|
|
150
|
+
| `max_dom_nodes` | 50,000 | DOM node cap (HTML/schema path) |
|
|
151
|
+
| `max_dom_depth` | 50 | DOM depth cap |
|
|
152
|
+
| `max_records` | 10,000 | schema records cap |
|
|
153
|
+
| `max_fields_per_record` | 100 | fields per schema record cap |
|
|
154
|
+
| `max_zip_entries` | 10,000 | office container entry cap |
|
|
155
|
+
| `max_zip_ratio` | 1,000 | office container compression-bomb ratio cap |
|
|
156
|
+
|
|
157
|
+
`RICH_TEXT_FLOOR = 200` is the minimum body-text length for `quality="high"` (requires a
|
|
158
|
+
main-content element and a non-lossy decode).
|
|
159
|
+
|
|
160
|
+
## Quality ladder
|
|
161
|
+
|
|
162
|
+
Closed vocabulary, evaluated by `_resolve_quality`:
|
|
163
|
+
|
|
164
|
+
| Grade | Meaning |
|
|
165
|
+
|---|---|
|
|
166
|
+
| `HIGH` | main-content element found, body text ≥ `RICH_TEXT_FLOOR`, decode not lossy |
|
|
167
|
+
| `MEDIUM` | body text present, but below the floor or a lossy decode |
|
|
168
|
+
| `LOW` | decoded text or structural content (links/forms/tables) present, no body text / lossy |
|
|
169
|
+
| `FAILED` | no extractable content, refused format, or input refused |
|
|
170
|
+
|
|
171
|
+
A clean non-HTML document with decoded text is `LOW` (no main-content concept). An empty
|
|
172
|
+
result is never above `LOW`.
|
|
173
|
+
|
|
174
|
+
## Flags vocabulary (degradation, not failure)
|
|
175
|
+
|
|
176
|
+
Flags describe degradation, never data loss. `extract` never returns partial data silently:
|
|
177
|
+
every flag is paired with the result it qualifies, and `truncated` is set whenever a cap
|
|
178
|
+
bit. Representative flags:
|
|
179
|
+
|
|
180
|
+
* `charset_lossy` — text had to be lossily re-encoded
|
|
181
|
+
* `links_refused` — one or more links refused by the URL policy
|
|
182
|
+
* `input_refused` — input exceeded `max_input_bytes`
|
|
183
|
+
* `format_refused` — content type not supported
|
|
184
|
+
* `selector_invalid` / `schema_required_missing` — schema selector or required field issue
|
|
185
|
+
* `records_truncated` / `fields_truncated` — a schema cap was hit
|
|
186
|
+
* `schema_dom_cap` — the DOM cap prevented schema extraction
|
|
187
|
+
* `pdf_encoding_partial` — PDF text sweep could not recover the full encoding
|
|
188
|
+
* `table_grid_degraded` / `nested_table_flattened` — a table was normalized, not dropped
|
|
189
|
+
|
|
190
|
+
Text is never discarded because markdown cannot express a shape: nested tables flatten into
|
|
191
|
+
the enclosing cell (pipes escaped, `nested_table_flattened` set), `colspan` repeats content
|
|
192
|
+
and `rowspan` carries it down, and the grid stays rectangular.
|
|
193
|
+
|
|
194
|
+
## Hard dependency set
|
|
195
|
+
|
|
196
|
+
`crawlutron-extract` ships no extras and is not stdlib-only. The hard dependencies are
|
|
197
|
+
imported plainly — a broken install fails at import, there is no lazy/fallback path:
|
|
198
|
+
|
|
199
|
+
* `crawlutron-safe-url` — `NormalizedUrl` and `HostPolicy`; its own closure is also hard:
|
|
200
|
+
* `regex` — bounded matcher
|
|
201
|
+
* `publicsuffixlist` — public suffix list
|
|
202
|
+
* `pdfminer.six` — PDF tier-1 text sweep
|
|
203
|
+
|
|
204
|
+
Credentials and URLs are object-invariant: a `NormalizedUrl` is the sole validation
|
|
205
|
+
chokepoint (it rejects userinfo), and secrets never surface unless intentionally consumed.
|
|
206
|
+
|
|
207
|
+
## Deliberate non-features
|
|
208
|
+
|
|
209
|
+
* No OCR — scanned or image-only PDFs/pages yield no text.
|
|
210
|
+
* No JS execution — script content is never executed; JS endpoints are surfaced as text or
|
|
211
|
+
links only, never evaluated.
|
|
212
|
+
* No network I/O — extraction is a pure `bytes` → `Extraction` function; DNS resolution and
|
|
213
|
+
host pinning are the caller's responsibility.
|
|
214
|
+
* No DOM for the streaming path — text formats (CSV/JSON/XML) are swept single-pass
|
|
215
|
+
without building a DOM; the HTML/schema path builds a bounded DOM under `max_dom_nodes`.
|
|
216
|
+
* PDF tables are reconstructed from layout rather than a full PDF object model — pdfminer.six
|
|
217
|
+
supplies the tier-1 text sweep; there is no general PDF DOM.
|
|
218
|
+
|
|
219
|
+
## Guarantees
|
|
220
|
+
|
|
221
|
+
Bounded (every list capped, input size refused), deterministic (same bytes ⇒ same result),
|
|
222
|
+
never raises (hostile input ⇒ `quality="failed"`), and closed (CSS and XPath subsets are
|
|
223
|
+
refused outside, never silently extended).
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Bench results — crawlutron-extract
|
|
2
|
+
|
|
3
|
+
Median-of-7 extraction timings from `packages/extract/bench/bench_extract.py`.
|
|
4
|
+
|
|
5
|
+
Machine: MacBook-Pro.local, Python 3.14.10 on darwin
|
|
6
|
+
Date: 2026-11-18
|
|
7
|
+
|
|
8
|
+
| Case | Input | Median (s) |
|
|
9
|
+
|---|---|---|
|
|
10
|
+
| HTML 100 KiB | ~100 KiB text/html | 0.007148 |
|
|
11
|
+
| HTML 1 MiB | ~1 MiB text/html | 0.074795 |
|
|
12
|
+
| DOCX small | 5 paragraphs | 0.000055 |
|
|
13
|
+
| DOCX large | 200 paragraphs | 0.000473 |
|
|
14
|
+
| PDF real (golden) | simple1.pdf (849 B) | 0.000024 |
|
|
15
|
+
| schema extraction | ~50 KiB HTML, 2 fields | 0.019998 |
|
|
16
|
+
|
|
17
|
+
Coarse timing gate: 1 MiB HTML under 2 s (observed 0.074795 s). This is the
|
|
18
|
+
suite's only timing assertion; timing is inherently flaky, so the bound is
|
|
19
|
+
deliberately coarse and no other timing assertions exist.
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Median-of-N benchmarks for crawlutron_extract extraction timings."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import statistics
|
|
6
|
+
import time
|
|
7
|
+
import zipfile
|
|
8
|
+
from io import BytesIO
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from crawlutron_extract import extract
|
|
12
|
+
from crawlutron_extract.schema import FieldKind, FieldSpec, Schema, extract_records
|
|
13
|
+
|
|
14
|
+
REPEATS = 7
|
|
15
|
+
DOCX_CT = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
|
|
16
|
+
GOLDEN = Path(__file__).resolve().parent.parent / "tests" / "golden"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _median_n(fn, n: int = REPEATS) -> float:
|
|
20
|
+
fn() # warmup
|
|
21
|
+
times: list[float] = []
|
|
22
|
+
for _ in range(n):
|
|
23
|
+
start = time.perf_counter()
|
|
24
|
+
fn()
|
|
25
|
+
times.append(time.perf_counter() - start)
|
|
26
|
+
return statistics.median(times)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _html_page(size_bytes: int) -> str:
|
|
30
|
+
"""Deterministic HTML of roughly ``size_bytes`` bytes."""
|
|
31
|
+
parts = ["<html><body><h1>Bench page</h1>"]
|
|
32
|
+
paragraph = "<p>The quick brown fox jumps over the lazy dog. </p>"
|
|
33
|
+
while sum(len(p.encode("utf-8")) for p in parts) < size_bytes:
|
|
34
|
+
parts.append(paragraph)
|
|
35
|
+
parts.append("</body></html>")
|
|
36
|
+
return "".join(parts)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _build_docx(paragraphs: int) -> bytes:
|
|
40
|
+
"""Minimal DOCX with ``paragraphs`` paragraphs (same shape as the test fixtures)."""
|
|
41
|
+
rows = "".join(
|
|
42
|
+
f"<w:p><w:r><w:t>bench paragraph {i}: alpha beta gamma delta epsilon</w:t></w:r></w:p>"
|
|
43
|
+
for i in range(paragraphs)
|
|
44
|
+
)
|
|
45
|
+
document = (
|
|
46
|
+
'<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">'
|
|
47
|
+
"<w:body>" + rows + "</w:body></w:document>"
|
|
48
|
+
)
|
|
49
|
+
buffer = BytesIO()
|
|
50
|
+
with zipfile.ZipFile(buffer, "w") as zf:
|
|
51
|
+
zf.writestr(
|
|
52
|
+
"[Content_Types].xml",
|
|
53
|
+
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
|
|
54
|
+
'<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"/>',
|
|
55
|
+
)
|
|
56
|
+
zf.writestr("word/document.xml", document)
|
|
57
|
+
return buffer.getvalue()
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _case(name: str, content_type: str, fn) -> tuple[str, str, float]:
|
|
61
|
+
elapsed = _median_n(fn)
|
|
62
|
+
return name, content_type, elapsed
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def main() -> None:
|
|
66
|
+
rows = []
|
|
67
|
+
html_100k = _html_page(100 * 1024).encode("utf-8")
|
|
68
|
+
rows.append(_case("HTML 100 KiB", "text/html", lambda: extract(html_100k, "text/html")))
|
|
69
|
+
html_1mib = _html_page(1024 * 1024).encode("utf-8")
|
|
70
|
+
rows.append(_case("HTML 1 MiB", "text/html", lambda: extract(html_1mib, "text/html")))
|
|
71
|
+
rows.append(_case("DOCX small", DOCX_CT, lambda: extract(_build_docx(5), DOCX_CT)))
|
|
72
|
+
rows.append(_case("DOCX large", DOCX_CT, lambda: extract(_build_docx(200), DOCX_CT)))
|
|
73
|
+
pdf_bytes = (GOLDEN / "simple1.pdf").read_bytes()
|
|
74
|
+
rows.append(_case("PDF real (golden)", "application/pdf", lambda: extract(pdf_bytes, "application/pdf")))
|
|
75
|
+
schema = Schema(
|
|
76
|
+
record="article",
|
|
77
|
+
fields=(
|
|
78
|
+
FieldSpec("title", "h1", FieldKind.TEXT),
|
|
79
|
+
FieldSpec("body", "p", FieldKind.TEXT, many=True),
|
|
80
|
+
),
|
|
81
|
+
)
|
|
82
|
+
rows.append(
|
|
83
|
+
_case(
|
|
84
|
+
"schema extraction",
|
|
85
|
+
"text/html",
|
|
86
|
+
lambda: extract_records(_html_page(50 * 1024), schema),
|
|
87
|
+
)
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
print("crawlutron-extract bench (median of", REPEATS, "repeats)")
|
|
91
|
+
print()
|
|
92
|
+
print(f"{'Case':<24} {'Content type':<48} {'median (s)'}")
|
|
93
|
+
print("-" * 78)
|
|
94
|
+
for name, ctype, elapsed in rows:
|
|
95
|
+
print(f"{name:<24} {ctype:<48} {elapsed:.6f}")
|
|
96
|
+
print()
|
|
97
|
+
print("Slowest:", max(rows, key=lambda r: r[2])[0])
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
if __name__ == "__main__":
|
|
101
|
+
main()
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "crawlutron-extract"
|
|
7
|
+
version = "0.5.0"
|
|
8
|
+
description = "Content extraction: HTML, office containers, text formats, PDF tier 1, bounded and deterministic"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.14"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
|
|
13
|
+
# crawlutron-safe-url is required for link admission (slice 5). Its own
|
|
14
|
+
# dependency closure includes regex (bounded matcher) and publicsuffixlist
|
|
15
|
+
# (public suffix list) as hard dependencies; the default install is not
|
|
16
|
+
# stdlib-only. pdfminer.six is a hard dependency for PDF tier-1 fallback.
|
|
17
|
+
dependencies = ["crawlutron-safe-url", "pdfminer.six==20260107"]
|
|
18
|
+
|
|
19
|
+
[tool.hatch.build.targets.wheel]
|
|
20
|
+
packages = ["src/crawlutron_extract"]
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""crawlutron_extract: stdlib-only content extraction, bounded and deterministic."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from crawlutron_extract._extract import DEFAULT_LIMITS, extract
|
|
6
|
+
from crawlutron_extract.css_select import SelectorError, validate_css
|
|
7
|
+
from crawlutron_extract.html_diagnostics import html_diagnostics
|
|
8
|
+
from crawlutron_extract.js_endpoints import extract_js_endpoints
|
|
9
|
+
from crawlutron_extract.limits import ExtractLimits
|
|
10
|
+
from crawlutron_extract.links import LinkRecord, extract_discovered_links
|
|
11
|
+
from crawlutron_extract.models import Extraction, Quality
|
|
12
|
+
from crawlutron_extract.pagination import detect_pagination
|
|
13
|
+
from crawlutron_extract.schema import FieldKind, FieldSpec, Schema, extract_records
|
|
14
|
+
from crawlutron_extract.sitemap import extract_sitemap_locs
|
|
15
|
+
from crawlutron_extract.xpath_select import validate_xpath
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"DEFAULT_LIMITS",
|
|
19
|
+
"ExtractLimits",
|
|
20
|
+
"Extraction",
|
|
21
|
+
"FieldKind",
|
|
22
|
+
"FieldSpec",
|
|
23
|
+
"Quality",
|
|
24
|
+
"Schema",
|
|
25
|
+
"extract",
|
|
26
|
+
"extract_discovered_links",
|
|
27
|
+
"extract_js_endpoints",
|
|
28
|
+
"extract_records",
|
|
29
|
+
"extract_sitemap_locs",
|
|
30
|
+
"html_diagnostics",
|
|
31
|
+
"LinkRecord",
|
|
32
|
+
"SelectorError",
|
|
33
|
+
"detect_pagination",
|
|
34
|
+
"validate_css",
|
|
35
|
+
"validate_xpath",
|
|
36
|
+
]
|