gleanly 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gleanly/__about__.py +1 -0
- gleanly/__init__.py +62 -0
- gleanly/clean/__init__.py +15 -0
- gleanly/clean/blocks.py +201 -0
- gleanly/clean/comments.py +98 -0
- gleanly/clean/html.py +773 -0
- gleanly/clean/metadata.py +110 -0
- gleanly/cli.py +174 -0
- gleanly/client.py +791 -0
- gleanly/compliance/__init__.py +15 -0
- gleanly/compliance/aitxt.py +139 -0
- gleanly/compliance/assess.py +136 -0
- gleanly/compliance/license.py +93 -0
- gleanly/compliance/pii.py +51 -0
- gleanly/compliance/tdmrep.py +120 -0
- gleanly/config.py +63 -0
- gleanly/errors.py +31 -0
- gleanly/fetch/__init__.py +9 -0
- gleanly/fetch/browser.py +50 -0
- gleanly/fetch/embedded.py +108 -0
- gleanly/fetch/http.py +110 -0
- gleanly/fetch/ratelimit.py +33 -0
- gleanly/fetch/robots.py +111 -0
- gleanly/fetch/router.py +35 -0
- gleanly/loader.py +33 -0
- gleanly/models.py +303 -0
- gleanly/ocr/__init__.py +1 -0
- gleanly/ocr/engine.py +66 -0
- gleanly/ocr/preprocess.py +144 -0
- gleanly/ocr/tesseract.py +162 -0
- gleanly/pdf/__init__.py +5 -0
- gleanly/pdf/check.py +113 -0
- gleanly/pdf/layout.py +298 -0
- gleanly/pdf/native.py +261 -0
- gleanly/py.typed +0 -0
- gleanly/quality/__init__.py +129 -0
- gleanly/quality/crosscheck.py +102 -0
- gleanly/search/__init__.py +209 -0
- gleanly/search/agreement.py +287 -0
- gleanly/search/backends.py +186 -0
- gleanly/search/fusion.py +57 -0
- gleanly-0.1.0a1.dist-info/METADATA +167 -0
- gleanly-0.1.0a1.dist-info/RECORD +46 -0
- gleanly-0.1.0a1.dist-info/WHEEL +4 -0
- gleanly-0.1.0a1.dist-info/entry_points.txt +2 -0
- gleanly-0.1.0a1.dist-info/licenses/LICENSE +202 -0
gleanly/__about__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0a1"
|
gleanly/__init__.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""All-in-one text acquisition for AI: URL/file -> clean Markdown with quality,
|
|
2
|
+
compliance and provenance metadata; web search with the same on every result.
|
|
3
|
+
|
|
4
|
+
>>> from gleanly import load, search
|
|
5
|
+
>>> doc = load("https://example.com")
|
|
6
|
+
>>> doc.markdown, doc.quality.score, doc.compliance.ai_use
|
|
7
|
+
>>> results = search("tide pools", backends=["wikipedia"])
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from gleanly.__about__ import __version__
|
|
11
|
+
from gleanly.client import Client
|
|
12
|
+
from gleanly.config import LoadOptions
|
|
13
|
+
from gleanly.errors import (
|
|
14
|
+
AcquisitionError,
|
|
15
|
+
BrowserUnavailable,
|
|
16
|
+
FetchError,
|
|
17
|
+
RobotsDisallowed,
|
|
18
|
+
UnsupportedSourceError,
|
|
19
|
+
)
|
|
20
|
+
from gleanly.loader import aload, load
|
|
21
|
+
from gleanly.models import (
|
|
22
|
+
Block,
|
|
23
|
+
BlockType,
|
|
24
|
+
ComplianceInfo,
|
|
25
|
+
Document,
|
|
26
|
+
FetchStep,
|
|
27
|
+
FetchTrace,
|
|
28
|
+
Metadata,
|
|
29
|
+
Provenance,
|
|
30
|
+
QualityReport,
|
|
31
|
+
SearchHit,
|
|
32
|
+
SearchResponse,
|
|
33
|
+
SearchResult,
|
|
34
|
+
)
|
|
35
|
+
from gleanly.search import asearch, search
|
|
36
|
+
|
|
37
|
+
__all__ = [
|
|
38
|
+
"AcquisitionError",
|
|
39
|
+
"Block",
|
|
40
|
+
"BlockType",
|
|
41
|
+
"BrowserUnavailable",
|
|
42
|
+
"Client",
|
|
43
|
+
"ComplianceInfo",
|
|
44
|
+
"Document",
|
|
45
|
+
"FetchError",
|
|
46
|
+
"FetchStep",
|
|
47
|
+
"FetchTrace",
|
|
48
|
+
"LoadOptions",
|
|
49
|
+
"Metadata",
|
|
50
|
+
"Provenance",
|
|
51
|
+
"QualityReport",
|
|
52
|
+
"RobotsDisallowed",
|
|
53
|
+
"SearchHit",
|
|
54
|
+
"SearchResponse",
|
|
55
|
+
"SearchResult",
|
|
56
|
+
"UnsupportedSourceError",
|
|
57
|
+
"__version__",
|
|
58
|
+
"aload",
|
|
59
|
+
"asearch",
|
|
60
|
+
"load",
|
|
61
|
+
"search",
|
|
62
|
+
]
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Main-content extraction, HTML -> Markdown, and page metadata."""
|
|
2
|
+
|
|
3
|
+
from gleanly.clean.html import Extraction, RawBlock, extract_html, markdown_to_plain, parse_html
|
|
4
|
+
from gleanly.clean.metadata import extract_metadata, iter_json_ld, json_ld_objects
|
|
5
|
+
|
|
6
|
+
__all__ = [
|
|
7
|
+
"Extraction",
|
|
8
|
+
"RawBlock",
|
|
9
|
+
"extract_html",
|
|
10
|
+
"extract_metadata",
|
|
11
|
+
"iter_json_ld",
|
|
12
|
+
"json_ld_objects",
|
|
13
|
+
"markdown_to_plain",
|
|
14
|
+
"parse_html",
|
|
15
|
+
]
|
gleanly/clean/blocks.py
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
"""Block-classifier extractor: gleanly's second, independent view of a page.
|
|
2
|
+
|
|
3
|
+
The main extractor (html.py) picks one content container, Readability-style. This one
|
|
4
|
+
shares nothing with it: it splits the page into text blocks at block-level tags and
|
|
5
|
+
classifies each block on its own (jusText/boilerpipe-style: length, link density,
|
|
6
|
+
sentence punctuation and its neighbours). It never looks at class names or containers.
|
|
7
|
+
|
|
8
|
+
It has two jobs:
|
|
9
|
+
|
|
10
|
+
- **Cross-check** (quality/crosscheck.py): words of the main extraction that this view
|
|
11
|
+
also kept are "supported"; that gives the extraction confidence.
|
|
12
|
+
- **Rescue**: when the main extractor finds (almost) nothing but this view finds plenty,
|
|
13
|
+
the page is laid out in a way container scoring does not handle, so this view is used.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import re
|
|
19
|
+
from dataclasses import dataclass
|
|
20
|
+
|
|
21
|
+
from lxml.html import HtmlElement
|
|
22
|
+
|
|
23
|
+
from gleanly.clean.html import (
|
|
24
|
+
BLOCK_TAGS,
|
|
25
|
+
HEADINGS,
|
|
26
|
+
SKIP_TAGS,
|
|
27
|
+
Extraction,
|
|
28
|
+
RawBlock,
|
|
29
|
+
_is_element,
|
|
30
|
+
_norm,
|
|
31
|
+
)
|
|
32
|
+
from gleanly.models import BlockType
|
|
33
|
+
|
|
34
|
+
_SENTENCE_END = re.compile(r"[.!?。!?]['\")\]”’]?(\s|$)") # noqa: RUF001 (CJK stops)
|
|
35
|
+
|
|
36
|
+
# Block classes. Tuned on dev splits only; results are flat across nearby values.
|
|
37
|
+
BAD_LINK_DENSITY = 0.4
|
|
38
|
+
GOOD_WORDS = 20
|
|
39
|
+
GOOD_LINK_DENSITY = 0.25
|
|
40
|
+
SHORT_WORDS = 7
|
|
41
|
+
# Rescue: use this view when the main extraction has at most RESCUE_WORDS words and this
|
|
42
|
+
# view has RESCUE_RATIO times more (twice as many if the main one is nearly empty).
|
|
43
|
+
RESCUE_WORDS = 150
|
|
44
|
+
RESCUE_RATIO = 6
|
|
45
|
+
NEARLY_EMPTY = 50
|
|
46
|
+
MIN_RESCUE_WORDS = 20
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass
|
|
50
|
+
class Segment:
|
|
51
|
+
el: HtmlElement # element whose own text this is (provenance)
|
|
52
|
+
text: str
|
|
53
|
+
link_chars: int
|
|
54
|
+
|
|
55
|
+
@property
|
|
56
|
+
def words(self) -> int:
|
|
57
|
+
return len(self.text.split())
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _page_root(root: HtmlElement) -> HtmlElement:
|
|
61
|
+
"""The <body>, unless broken markup left the text outside it."""
|
|
62
|
+
body = root.find("body")
|
|
63
|
+
if body is not None and len(body.text_content().strip()) > 50:
|
|
64
|
+
return body
|
|
65
|
+
return root
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def segments(root: HtmlElement, skip: frozenset[str] | set[str] = SKIP_TAGS) -> list[Segment]:
|
|
69
|
+
"""Split the page into text blocks at block-level tags, in document order.
|
|
70
|
+
|
|
71
|
+
Elements whose tag is in ``skip`` are left out together with their text.
|
|
72
|
+
"""
|
|
73
|
+
out: list[Segment] = []
|
|
74
|
+
|
|
75
|
+
def flush(el: HtmlElement, parts: list[str], links: int) -> None:
|
|
76
|
+
text = _norm(" ".join(parts)).strip()
|
|
77
|
+
if text:
|
|
78
|
+
out.append(Segment(el, text, links))
|
|
79
|
+
|
|
80
|
+
def rec(el: HtmlElement) -> None:
|
|
81
|
+
parts, links = [el.text or ""], 0
|
|
82
|
+
for child in el:
|
|
83
|
+
if _is_element(child) and child.tag not in skip:
|
|
84
|
+
if child.tag in BLOCK_TAGS:
|
|
85
|
+
flush(el, parts, links)
|
|
86
|
+
parts, links = [], 0
|
|
87
|
+
rec(child)
|
|
88
|
+
else:
|
|
89
|
+
parts.append(child.text_content())
|
|
90
|
+
anchors = [child] if child.tag == "a" else child.iter("a")
|
|
91
|
+
links += sum(len(_norm(a.text_content()).strip()) for a in anchors)
|
|
92
|
+
parts.append(child.tail or "")
|
|
93
|
+
flush(el, parts, links)
|
|
94
|
+
|
|
95
|
+
rec(_page_root(root))
|
|
96
|
+
return out
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def labels(segs: list[Segment]) -> list[str]:
|
|
100
|
+
"""Context-free class of each block: "good" (prose), "near", "short" or "bad"."""
|
|
101
|
+
out: list[str] = []
|
|
102
|
+
for seg in segs:
|
|
103
|
+
density = seg.link_chars / len(seg.text)
|
|
104
|
+
if density > BAD_LINK_DENSITY:
|
|
105
|
+
out.append("bad")
|
|
106
|
+
elif (
|
|
107
|
+
seg.words >= GOOD_WORDS
|
|
108
|
+
and density < GOOD_LINK_DENSITY
|
|
109
|
+
and _SENTENCE_END.search(seg.text)
|
|
110
|
+
):
|
|
111
|
+
out.append("good")
|
|
112
|
+
elif seg.words < SHORT_WORDS:
|
|
113
|
+
out.append("short")
|
|
114
|
+
else:
|
|
115
|
+
out.append("near")
|
|
116
|
+
return out
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def classify(segs: list[Segment]) -> list[bool]:
|
|
120
|
+
"""True for blocks this method considers main content."""
|
|
121
|
+
labels_ = labels(segs)
|
|
122
|
+
|
|
123
|
+
def nearest(i: int, step: int) -> str:
|
|
124
|
+
j = i + step
|
|
125
|
+
while 0 <= j < len(labels_):
|
|
126
|
+
if labels_[j] != "short":
|
|
127
|
+
return labels_[j]
|
|
128
|
+
j += step
|
|
129
|
+
return "bad"
|
|
130
|
+
|
|
131
|
+
keep: list[bool] = []
|
|
132
|
+
for i, label in enumerate(labels_):
|
|
133
|
+
if label == "good":
|
|
134
|
+
keep.append(True)
|
|
135
|
+
elif label == "near": # medium block next to real prose
|
|
136
|
+
keep.append("good" in (nearest(i, -1), nearest(i, 1)))
|
|
137
|
+
elif label == "short": # short block inside a run of prose
|
|
138
|
+
keep.append(nearest(i, -1) == "good" and nearest(i, 1) == "good")
|
|
139
|
+
else:
|
|
140
|
+
keep.append(False)
|
|
141
|
+
return keep
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def content_segments(root: HtmlElement) -> list[Segment]:
|
|
145
|
+
segs = segments(root)
|
|
146
|
+
return [s for s, k in zip(segs, classify(segs), strict=True) if k]
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def reference_text(root: HtmlElement) -> str:
|
|
150
|
+
"""This view of the page's main content, as plain text."""
|
|
151
|
+
return "\n".join(s.text for s in content_segments(root))
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def prose_text(root: HtmlElement) -> str:
|
|
155
|
+
"""Only the blocks that read as prose on their own (long, few links, sentences)."""
|
|
156
|
+
segs = segments(root)
|
|
157
|
+
return "\n".join(s.text for s, lab in zip(segs, labels(segs), strict=True) if lab == "good")
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def extract_blocks(root: HtmlElement, method: str) -> Extraction:
|
|
161
|
+
"""Run this view as a full extractor, with an XPath for every block."""
|
|
162
|
+
tree = root.getroottree()
|
|
163
|
+
blocks: list[RawBlock] = []
|
|
164
|
+
link_chars = 0
|
|
165
|
+
for seg in content_segments(root):
|
|
166
|
+
path = tree.getpath(seg.el)
|
|
167
|
+
if seg.el.tag in HEADINGS:
|
|
168
|
+
level = int(seg.el.tag[1])
|
|
169
|
+
blocks.append(RawBlock(BlockType.HEADING, f"{'#' * level} {seg.text}", path, level))
|
|
170
|
+
else:
|
|
171
|
+
blocks.append(RawBlock(BlockType.PARAGRAPH, seg.text, path))
|
|
172
|
+
link_chars += seg.link_chars
|
|
173
|
+
page_chars = len(_norm(_page_root(root).text_content()).strip())
|
|
174
|
+
return Extraction(
|
|
175
|
+
blocks, method, link_chars=link_chars, page_chars=page_chars, extractor="blocks"
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def rescue(extraction: Extraction, root: HtmlElement) -> Extraction | None:
|
|
180
|
+
"""The block view, if the main extraction missed most of the page's prose."""
|
|
181
|
+
words = extraction.word_count
|
|
182
|
+
if words > RESCUE_WORDS or extraction.extractor != "main" or extraction.method == "embedded":
|
|
183
|
+
return None
|
|
184
|
+
candidate = extract_blocks(root, extraction.method)
|
|
185
|
+
ratio = 2 if words <= NEARLY_EMPTY else RESCUE_RATIO
|
|
186
|
+
if candidate.word_count > max(words * ratio, MIN_RESCUE_WORDS):
|
|
187
|
+
return candidate
|
|
188
|
+
return None
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
__all__ = [
|
|
192
|
+
"Segment",
|
|
193
|
+
"classify",
|
|
194
|
+
"content_segments",
|
|
195
|
+
"extract_blocks",
|
|
196
|
+
"labels",
|
|
197
|
+
"prose_text",
|
|
198
|
+
"reference_text",
|
|
199
|
+
"rescue",
|
|
200
|
+
"segments",
|
|
201
|
+
]
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Optional reader comments (``LoadOptions.include_comments``).
|
|
2
|
+
|
|
3
|
+
Main-content extraction drops comment sections on purpose: for most AI uses they are
|
|
4
|
+
noise. Some users want them (forum threads, discussion datasets), and benchmarks
|
|
5
|
+
disagree on whether they count as content, so they are opt-in.
|
|
6
|
+
|
|
7
|
+
Method: comment sections are found by their class/id names (``comments``, ``disqus`` ...),
|
|
8
|
+
then only the blocks that the block classifier (clean/blocks.py) labels as prose are
|
|
9
|
+
kept. That drops the furniture around each comment ("Reply", dates, avatars, vote
|
|
10
|
+
counts, the reply form) without site-specific rules. Comment blocks carry
|
|
11
|
+
``role="comment"`` so they can be filtered out again downstream.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import re
|
|
17
|
+
from collections.abc import Callable
|
|
18
|
+
|
|
19
|
+
from lxml.html import HtmlElement
|
|
20
|
+
|
|
21
|
+
from gleanly.clean.blocks import labels, segments
|
|
22
|
+
from gleanly.clean.html import HEADINGS, SKIP_TAGS, Extraction, RawBlock, _is_element
|
|
23
|
+
from gleanly.models import BlockType
|
|
24
|
+
|
|
25
|
+
COMMENT_WORDS = {"comment", "comments", "commentlist", "disqus", "discussion", "replies"}
|
|
26
|
+
# Block classes kept inside a comment section: "near" adds medium-length comments.
|
|
27
|
+
KEEP_LABELS = {"good", "near"}
|
|
28
|
+
_SPLIT = re.compile(r"[\s_\-]+")
|
|
29
|
+
_SKIP = SKIP_TAGS | {"form"}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _is_comment_section(el: HtmlElement) -> bool:
|
|
33
|
+
hint = f"{el.get('class') or ''} {el.get('id') or ''}".lower()
|
|
34
|
+
return bool(hint.strip()) and bool(set(_SPLIT.split(hint)) & COMMENT_WORDS)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _hidden(el: HtmlElement) -> bool:
|
|
38
|
+
for node in (el, *el.iterancestors()):
|
|
39
|
+
style = (node.get("style") or "").replace(" ", "").lower()
|
|
40
|
+
if node.get("hidden") is not None or "display:none" in style:
|
|
41
|
+
return True
|
|
42
|
+
return False
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def comment_sections(
|
|
46
|
+
body: HtmlElement,
|
|
47
|
+
container: HtmlElement | None,
|
|
48
|
+
already_extracted: Callable[[HtmlElement], bool] = lambda _: False,
|
|
49
|
+
) -> list[HtmlElement]:
|
|
50
|
+
"""Outermost comment sections that are not the article and not already extracted."""
|
|
51
|
+
found: list[HtmlElement] = []
|
|
52
|
+
for el in body.iter():
|
|
53
|
+
if not _is_element(el) or not _is_comment_section(el):
|
|
54
|
+
continue
|
|
55
|
+
if container is not None and (
|
|
56
|
+
el is container or any(a is el for a in container.iterancestors())
|
|
57
|
+
):
|
|
58
|
+
continue # e.g. WordPress' <body class="comments-open"> wraps the article too
|
|
59
|
+
if any(f is a for f in found for a in el.iterancestors()):
|
|
60
|
+
continue # inside a section we already have
|
|
61
|
+
if already_extracted(el) or _hidden(el):
|
|
62
|
+
continue
|
|
63
|
+
found.append(el)
|
|
64
|
+
return found
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def comment_blocks(sections: list[HtmlElement]) -> list[RawBlock]:
|
|
68
|
+
out: list[RawBlock] = []
|
|
69
|
+
for section in sections:
|
|
70
|
+
tree = section.getroottree()
|
|
71
|
+
segs = segments(section, _SKIP) # forms: "Leave a reply" and its notes
|
|
72
|
+
for seg, label in zip(segs, labels(segs), strict=True):
|
|
73
|
+
if label not in KEEP_LABELS:
|
|
74
|
+
continue
|
|
75
|
+
path = tree.getpath(seg.el)
|
|
76
|
+
if seg.el.tag in HEADINGS: # "2 thoughts on ...": keep the section's structure
|
|
77
|
+
level = int(seg.el.tag[1])
|
|
78
|
+
text = f"{'#' * level} {seg.text}"
|
|
79
|
+
out.append(RawBlock(BlockType.HEADING, text, path, level, role="comment"))
|
|
80
|
+
else:
|
|
81
|
+
out.append(RawBlock(BlockType.PARAGRAPH, seg.text, path, role="comment"))
|
|
82
|
+
return out
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def with_comments(extraction: Extraction, root: HtmlElement) -> Extraction:
|
|
86
|
+
"""Add the page's comments to text that came from elsewhere (e.g. embedded JSON-LD)."""
|
|
87
|
+
if any(b.role == "comment" for b in extraction.blocks):
|
|
88
|
+
return extraction
|
|
89
|
+
body = root.find("body")
|
|
90
|
+
found = comment_blocks(comment_sections(body if body is not None else root, None))
|
|
91
|
+
if not found:
|
|
92
|
+
return extraction
|
|
93
|
+
extraction.blocks = [*extraction.blocks, *found]
|
|
94
|
+
extraction.refresh()
|
|
95
|
+
return extraction
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
__all__ = ["COMMENT_WORDS", "comment_blocks", "comment_sections", "with_comments"]
|