gleanly 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. gleanly/__about__.py +1 -0
  2. gleanly/__init__.py +62 -0
  3. gleanly/clean/__init__.py +15 -0
  4. gleanly/clean/blocks.py +201 -0
  5. gleanly/clean/comments.py +98 -0
  6. gleanly/clean/html.py +773 -0
  7. gleanly/clean/metadata.py +110 -0
  8. gleanly/cli.py +174 -0
  9. gleanly/client.py +791 -0
  10. gleanly/compliance/__init__.py +15 -0
  11. gleanly/compliance/aitxt.py +139 -0
  12. gleanly/compliance/assess.py +136 -0
  13. gleanly/compliance/license.py +93 -0
  14. gleanly/compliance/pii.py +51 -0
  15. gleanly/compliance/tdmrep.py +120 -0
  16. gleanly/config.py +63 -0
  17. gleanly/errors.py +31 -0
  18. gleanly/fetch/__init__.py +9 -0
  19. gleanly/fetch/browser.py +50 -0
  20. gleanly/fetch/embedded.py +108 -0
  21. gleanly/fetch/http.py +110 -0
  22. gleanly/fetch/ratelimit.py +33 -0
  23. gleanly/fetch/robots.py +111 -0
  24. gleanly/fetch/router.py +35 -0
  25. gleanly/loader.py +33 -0
  26. gleanly/models.py +303 -0
  27. gleanly/ocr/__init__.py +1 -0
  28. gleanly/ocr/engine.py +66 -0
  29. gleanly/ocr/preprocess.py +144 -0
  30. gleanly/ocr/tesseract.py +162 -0
  31. gleanly/pdf/__init__.py +5 -0
  32. gleanly/pdf/check.py +113 -0
  33. gleanly/pdf/layout.py +298 -0
  34. gleanly/pdf/native.py +261 -0
  35. gleanly/py.typed +0 -0
  36. gleanly/quality/__init__.py +129 -0
  37. gleanly/quality/crosscheck.py +102 -0
  38. gleanly/search/__init__.py +209 -0
  39. gleanly/search/agreement.py +287 -0
  40. gleanly/search/backends.py +186 -0
  41. gleanly/search/fusion.py +57 -0
  42. gleanly-0.1.0a1.dist-info/METADATA +167 -0
  43. gleanly-0.1.0a1.dist-info/RECORD +46 -0
  44. gleanly-0.1.0a1.dist-info/WHEEL +4 -0
  45. gleanly-0.1.0a1.dist-info/entry_points.txt +2 -0
  46. gleanly-0.1.0a1.dist-info/licenses/LICENSE +202 -0
gleanly/__about__.py ADDED
@@ -0,0 +1 @@
1
+ __version__ = "0.1.0a1"
gleanly/__init__.py ADDED
@@ -0,0 +1,62 @@
1
+ """All-in-one text acquisition for AI: URL/file -> clean Markdown with quality,
2
+ compliance and provenance metadata; web search with the same on every result.
3
+
4
+ >>> from gleanly import load, search
5
+ >>> doc = load("https://example.com")
6
+ >>> doc.markdown, doc.quality.score, doc.compliance.ai_use
7
+ >>> results = search("tide pools", backends=["wikipedia"])
8
+ """
9
+
10
+ from gleanly.__about__ import __version__
11
+ from gleanly.client import Client
12
+ from gleanly.config import LoadOptions
13
+ from gleanly.errors import (
14
+ AcquisitionError,
15
+ BrowserUnavailable,
16
+ FetchError,
17
+ RobotsDisallowed,
18
+ UnsupportedSourceError,
19
+ )
20
+ from gleanly.loader import aload, load
21
+ from gleanly.models import (
22
+ Block,
23
+ BlockType,
24
+ ComplianceInfo,
25
+ Document,
26
+ FetchStep,
27
+ FetchTrace,
28
+ Metadata,
29
+ Provenance,
30
+ QualityReport,
31
+ SearchHit,
32
+ SearchResponse,
33
+ SearchResult,
34
+ )
35
+ from gleanly.search import asearch, search
36
+
37
+ __all__ = [
38
+ "AcquisitionError",
39
+ "Block",
40
+ "BlockType",
41
+ "BrowserUnavailable",
42
+ "Client",
43
+ "ComplianceInfo",
44
+ "Document",
45
+ "FetchError",
46
+ "FetchStep",
47
+ "FetchTrace",
48
+ "LoadOptions",
49
+ "Metadata",
50
+ "Provenance",
51
+ "QualityReport",
52
+ "RobotsDisallowed",
53
+ "SearchHit",
54
+ "SearchResponse",
55
+ "SearchResult",
56
+ "UnsupportedSourceError",
57
+ "__version__",
58
+ "aload",
59
+ "asearch",
60
+ "load",
61
+ "search",
62
+ ]
@@ -0,0 +1,15 @@
1
+ """Main-content extraction, HTML -> Markdown, and page metadata."""
2
+
3
+ from gleanly.clean.html import Extraction, RawBlock, extract_html, markdown_to_plain, parse_html
4
+ from gleanly.clean.metadata import extract_metadata, iter_json_ld, json_ld_objects
5
+
6
+ __all__ = [
7
+ "Extraction",
8
+ "RawBlock",
9
+ "extract_html",
10
+ "extract_metadata",
11
+ "iter_json_ld",
12
+ "json_ld_objects",
13
+ "markdown_to_plain",
14
+ "parse_html",
15
+ ]
@@ -0,0 +1,201 @@
1
+ """Block-classifier extractor: gleanly's second, independent view of a page.
2
+
3
+ The main extractor (html.py) picks one content container, Readability-style. This one
4
+ shares nothing with it: it splits the page into text blocks at block-level tags and
5
+ classifies each block on its own (jusText/boilerpipe-style: length, link density,
6
+ sentence punctuation and its neighbours). It never looks at class names or containers.
7
+
8
+ It has two jobs:
9
+
10
+ - **Cross-check** (quality/crosscheck.py): words of the main extraction that this view
11
+ also kept are "supported"; that gives the extraction confidence.
12
+ - **Rescue**: when the main extractor finds (almost) nothing but this view finds plenty,
13
+ the page is laid out in a way container scoring does not handle, so this view is used.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import re
19
+ from dataclasses import dataclass
20
+
21
+ from lxml.html import HtmlElement
22
+
23
+ from gleanly.clean.html import (
24
+ BLOCK_TAGS,
25
+ HEADINGS,
26
+ SKIP_TAGS,
27
+ Extraction,
28
+ RawBlock,
29
+ _is_element,
30
+ _norm,
31
+ )
32
+ from gleanly.models import BlockType
33
+
34
+ _SENTENCE_END = re.compile(r"[.!?。!?]['\")\]”’]?(\s|$)") # noqa: RUF001 (CJK stops)
35
+
36
+ # Block classes. Tuned on dev splits only; results are flat across nearby values.
37
+ BAD_LINK_DENSITY = 0.4
38
+ GOOD_WORDS = 20
39
+ GOOD_LINK_DENSITY = 0.25
40
+ SHORT_WORDS = 7
41
+ # Rescue: use this view when the main extraction has at most RESCUE_WORDS words and this
42
+ # view has RESCUE_RATIO times more (twice as many if the main one is nearly empty).
43
+ RESCUE_WORDS = 150
44
+ RESCUE_RATIO = 6
45
+ NEARLY_EMPTY = 50
46
+ MIN_RESCUE_WORDS = 20
47
+
48
+
49
+ @dataclass
50
+ class Segment:
51
+ el: HtmlElement # element whose own text this is (provenance)
52
+ text: str
53
+ link_chars: int
54
+
55
+ @property
56
+ def words(self) -> int:
57
+ return len(self.text.split())
58
+
59
+
60
+ def _page_root(root: HtmlElement) -> HtmlElement:
61
+ """The <body>, unless broken markup left the text outside it."""
62
+ body = root.find("body")
63
+ if body is not None and len(body.text_content().strip()) > 50:
64
+ return body
65
+ return root
66
+
67
+
68
+ def segments(root: HtmlElement, skip: frozenset[str] | set[str] = SKIP_TAGS) -> list[Segment]:
69
+ """Split the page into text blocks at block-level tags, in document order.
70
+
71
+ Elements whose tag is in ``skip`` are left out together with their text.
72
+ """
73
+ out: list[Segment] = []
74
+
75
+ def flush(el: HtmlElement, parts: list[str], links: int) -> None:
76
+ text = _norm(" ".join(parts)).strip()
77
+ if text:
78
+ out.append(Segment(el, text, links))
79
+
80
+ def rec(el: HtmlElement) -> None:
81
+ parts, links = [el.text or ""], 0
82
+ for child in el:
83
+ if _is_element(child) and child.tag not in skip:
84
+ if child.tag in BLOCK_TAGS:
85
+ flush(el, parts, links)
86
+ parts, links = [], 0
87
+ rec(child)
88
+ else:
89
+ parts.append(child.text_content())
90
+ anchors = [child] if child.tag == "a" else child.iter("a")
91
+ links += sum(len(_norm(a.text_content()).strip()) for a in anchors)
92
+ parts.append(child.tail or "")
93
+ flush(el, parts, links)
94
+
95
+ rec(_page_root(root))
96
+ return out
97
+
98
+
99
+ def labels(segs: list[Segment]) -> list[str]:
100
+ """Context-free class of each block: "good" (prose), "near", "short" or "bad"."""
101
+ out: list[str] = []
102
+ for seg in segs:
103
+ density = seg.link_chars / len(seg.text)
104
+ if density > BAD_LINK_DENSITY:
105
+ out.append("bad")
106
+ elif (
107
+ seg.words >= GOOD_WORDS
108
+ and density < GOOD_LINK_DENSITY
109
+ and _SENTENCE_END.search(seg.text)
110
+ ):
111
+ out.append("good")
112
+ elif seg.words < SHORT_WORDS:
113
+ out.append("short")
114
+ else:
115
+ out.append("near")
116
+ return out
117
+
118
+
119
+ def classify(segs: list[Segment]) -> list[bool]:
120
+ """True for blocks this method considers main content."""
121
+ labels_ = labels(segs)
122
+
123
+ def nearest(i: int, step: int) -> str:
124
+ j = i + step
125
+ while 0 <= j < len(labels_):
126
+ if labels_[j] != "short":
127
+ return labels_[j]
128
+ j += step
129
+ return "bad"
130
+
131
+ keep: list[bool] = []
132
+ for i, label in enumerate(labels_):
133
+ if label == "good":
134
+ keep.append(True)
135
+ elif label == "near": # medium block next to real prose
136
+ keep.append("good" in (nearest(i, -1), nearest(i, 1)))
137
+ elif label == "short": # short block inside a run of prose
138
+ keep.append(nearest(i, -1) == "good" and nearest(i, 1) == "good")
139
+ else:
140
+ keep.append(False)
141
+ return keep
142
+
143
+
144
+ def content_segments(root: HtmlElement) -> list[Segment]:
145
+ segs = segments(root)
146
+ return [s for s, k in zip(segs, classify(segs), strict=True) if k]
147
+
148
+
149
+ def reference_text(root: HtmlElement) -> str:
150
+ """This view of the page's main content, as plain text."""
151
+ return "\n".join(s.text for s in content_segments(root))
152
+
153
+
154
+ def prose_text(root: HtmlElement) -> str:
155
+ """Only the blocks that read as prose on their own (long, few links, sentences)."""
156
+ segs = segments(root)
157
+ return "\n".join(s.text for s, lab in zip(segs, labels(segs), strict=True) if lab == "good")
158
+
159
+
160
+ def extract_blocks(root: HtmlElement, method: str) -> Extraction:
161
+ """Run this view as a full extractor, with an XPath for every block."""
162
+ tree = root.getroottree()
163
+ blocks: list[RawBlock] = []
164
+ link_chars = 0
165
+ for seg in content_segments(root):
166
+ path = tree.getpath(seg.el)
167
+ if seg.el.tag in HEADINGS:
168
+ level = int(seg.el.tag[1])
169
+ blocks.append(RawBlock(BlockType.HEADING, f"{'#' * level} {seg.text}", path, level))
170
+ else:
171
+ blocks.append(RawBlock(BlockType.PARAGRAPH, seg.text, path))
172
+ link_chars += seg.link_chars
173
+ page_chars = len(_norm(_page_root(root).text_content()).strip())
174
+ return Extraction(
175
+ blocks, method, link_chars=link_chars, page_chars=page_chars, extractor="blocks"
176
+ )
177
+
178
+
179
+ def rescue(extraction: Extraction, root: HtmlElement) -> Extraction | None:
180
+ """The block view, if the main extraction missed most of the page's prose."""
181
+ words = extraction.word_count
182
+ if words > RESCUE_WORDS or extraction.extractor != "main" or extraction.method == "embedded":
183
+ return None
184
+ candidate = extract_blocks(root, extraction.method)
185
+ ratio = 2 if words <= NEARLY_EMPTY else RESCUE_RATIO
186
+ if candidate.word_count > max(words * ratio, MIN_RESCUE_WORDS):
187
+ return candidate
188
+ return None
189
+
190
+
191
+ __all__ = [
192
+ "Segment",
193
+ "classify",
194
+ "content_segments",
195
+ "extract_blocks",
196
+ "labels",
197
+ "prose_text",
198
+ "reference_text",
199
+ "rescue",
200
+ "segments",
201
+ ]
@@ -0,0 +1,98 @@
1
+ """Optional reader comments (``LoadOptions.include_comments``).
2
+
3
+ Main-content extraction drops comment sections on purpose: for most AI uses they are
4
+ noise. Some users want them (forum threads, discussion datasets), and benchmarks
5
+ disagree on whether they count as content, so they are opt-in.
6
+
7
+ Method: comment sections are found by their class/id names (``comments``, ``disqus`` ...),
8
+ then only the blocks that the block classifier (clean/blocks.py) labels as prose are
9
+ kept. That drops the furniture around each comment ("Reply", dates, avatars, vote
10
+ counts, the reply form) without site-specific rules. Comment blocks carry
11
+ ``role="comment"`` so they can be filtered out again downstream.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import re
17
+ from collections.abc import Callable
18
+
19
+ from lxml.html import HtmlElement
20
+
21
+ from gleanly.clean.blocks import labels, segments
22
+ from gleanly.clean.html import HEADINGS, SKIP_TAGS, Extraction, RawBlock, _is_element
23
+ from gleanly.models import BlockType
24
+
25
+ COMMENT_WORDS = {"comment", "comments", "commentlist", "disqus", "discussion", "replies"}
26
+ # Block classes kept inside a comment section: "near" adds medium-length comments.
27
+ KEEP_LABELS = {"good", "near"}
28
+ _SPLIT = re.compile(r"[\s_\-]+")
29
+ _SKIP = SKIP_TAGS | {"form"}
30
+
31
+
32
+ def _is_comment_section(el: HtmlElement) -> bool:
33
+ hint = f"{el.get('class') or ''} {el.get('id') or ''}".lower()
34
+ return bool(hint.strip()) and bool(set(_SPLIT.split(hint)) & COMMENT_WORDS)
35
+
36
+
37
+ def _hidden(el: HtmlElement) -> bool:
38
+ for node in (el, *el.iterancestors()):
39
+ style = (node.get("style") or "").replace(" ", "").lower()
40
+ if node.get("hidden") is not None or "display:none" in style:
41
+ return True
42
+ return False
43
+
44
+
45
+ def comment_sections(
46
+ body: HtmlElement,
47
+ container: HtmlElement | None,
48
+ already_extracted: Callable[[HtmlElement], bool] = lambda _: False,
49
+ ) -> list[HtmlElement]:
50
+ """Outermost comment sections that are not the article and not already extracted."""
51
+ found: list[HtmlElement] = []
52
+ for el in body.iter():
53
+ if not _is_element(el) or not _is_comment_section(el):
54
+ continue
55
+ if container is not None and (
56
+ el is container or any(a is el for a in container.iterancestors())
57
+ ):
58
+ continue # e.g. WordPress' <body class="comments-open"> wraps the article too
59
+ if any(f is a for f in found for a in el.iterancestors()):
60
+ continue # inside a section we already have
61
+ if already_extracted(el) or _hidden(el):
62
+ continue
63
+ found.append(el)
64
+ return found
65
+
66
+
67
+ def comment_blocks(sections: list[HtmlElement]) -> list[RawBlock]:
68
+ out: list[RawBlock] = []
69
+ for section in sections:
70
+ tree = section.getroottree()
71
+ segs = segments(section, _SKIP) # forms: "Leave a reply" and its notes
72
+ for seg, label in zip(segs, labels(segs), strict=True):
73
+ if label not in KEEP_LABELS:
74
+ continue
75
+ path = tree.getpath(seg.el)
76
+ if seg.el.tag in HEADINGS: # "2 thoughts on ...": keep the section's structure
77
+ level = int(seg.el.tag[1])
78
+ text = f"{'#' * level} {seg.text}"
79
+ out.append(RawBlock(BlockType.HEADING, text, path, level, role="comment"))
80
+ else:
81
+ out.append(RawBlock(BlockType.PARAGRAPH, seg.text, path, role="comment"))
82
+ return out
83
+
84
+
85
+ def with_comments(extraction: Extraction, root: HtmlElement) -> Extraction:
86
+ """Add the page's comments to text that came from elsewhere (e.g. embedded JSON-LD)."""
87
+ if any(b.role == "comment" for b in extraction.blocks):
88
+ return extraction
89
+ body = root.find("body")
90
+ found = comment_blocks(comment_sections(body if body is not None else root, None))
91
+ if not found:
92
+ return extraction
93
+ extraction.blocks = [*extraction.blocks, *found]
94
+ extraction.refresh()
95
+ return extraction
96
+
97
+
98
+ __all__ = ["COMMENT_WORDS", "comment_blocks", "comment_sections", "with_comments"]