python-corekit 0.4.1__py3-none-any.whl → 0.5.0__py3-none-any.whl

corekit/connections/sql/connection.py

@@ -330,15 +330,19 @@ class SQLConnection(Connectable):
         """
         return list(self._exec(query.build()))
 
-    def delete_many_by_query(self, query: Query) -> None:
+    def delete_many_by_query(self, query: Query) -> int:
         """
         Delete every row matching a Query.
+
+        Returns the number of rows deleted.
         """
         self.require_writable()
         self._start_session()
-        for item in self.fetch_many_by_query(query):
+        items = self.fetch_many_by_query(query)
+        for item in items:
             self.session.delete(item)
         self.session.commit()
+        return len(items)
 
     def execute(self, operation: Operation) -> Any:
         """

corekit/html/__init__.py

@@ -0,0 +1,22 @@
+"""
+HTML parsing and crawling helpers.
+"""
+
+from corekit.html.client import WebscrapeClient
+from corekit.html.models import LinkItem, Page, PageMetadata
+from corekit.html.parser import HtmlParser
+from corekit.html.scraper import Scraper
+from corekit.html.settings import CleanSettings, LinkFilterSettings
+from corekit.html.state import ScraperState
+
+__all__ = [
+    "CleanSettings",
+    "HtmlParser",
+    "LinkFilterSettings",
+    "LinkItem",
+    "Page",
+    "PageMetadata",
+    "Scraper",
+    "ScraperState",
+    "WebscrapeClient",
+]

corekit/html/client.py

@@ -0,0 +1,22 @@
+from typing import Any
+
+from corekit.http import BaseHttpClient
+
+
+class WebscrapeClient(BaseHttpClient):
+    """
+    HTTP client with browser-like default headers for scraping.
+
+    Inherits GET-only ``supported_methods`` from ``BaseHttpClient``.
+    """
+
+    def __init__(self, **kwargs: Any) -> None:
+        kwargs.setdefault("strict", False)
+        super().__init__(**kwargs)
+
+    @property
+    def headers(self) -> dict[str, str]:
+        return {
+            "Accept": "text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:156.0) Gecko/20100101 Firefox/156.0",
+        }

corekit/html/models.py

@@ -0,0 +1,20 @@
+from dataclasses import dataclass, field
+
+
+@dataclass(slots=True)
+class PageMetadata:
+    title: str = ""
+    base_url: str | None = None
+    meta: dict[str, str] = field(default_factory=dict)
+
+
+@dataclass(slots=True)
+class LinkItem:
+    href: str
+    text: str | None = None
+
+
+@dataclass(slots=True)
+class Page:
+    metadata: PageMetadata
+    content: str

corekit/html/parser.py

@@ -0,0 +1,272 @@
+from __future__ import annotations
+
+from typing import Any, Iterator
+from urllib.parse import urldefrag, urljoin, urlparse, urlunparse
+
+from selectolax.lexbor import LexborHTMLParser
+
+from corekit.exceptions import InternalCoreException
+from corekit.exceptions.enum import Retryability
+from corekit.html.models import LinkItem, PageMetadata
+from corekit.html.settings import CleanSettings, LinkFilterSettings
+from corekit.observability import Loggable
+from corekit.utils import iterator, parse_string, safe_dict, safe_string
+
+_SKIP_SCHEMES: frozenset[str] = frozenset({"", "javascript", "mailto", "tel", "data", "blob", "about"})
+
+
+class HtmlParser(Loggable):
+    def __init__(
+        self,
+        html: str | bytes,
+        url: str,
+        *,
+        clean_settings: CleanSettings | None = None,
+        link_settings: LinkFilterSettings | None = None,
+    ) -> None:
+        super().__init__()
+        self._html = html
+        self._url = url
+        self._clean_settings = clean_settings or CleanSettings.defaults()
+        self._link_settings = link_settings
+        self._original = LexborHTMLParser(html)
+        self._current = LexborHTMLParser(html)
+
+    def reset(self) -> None:
+        """
+        Resets the current working tree back to the pristine original state
+        """
+        self._current = LexborHTMLParser(self._html)
+
+    def clone(self) -> HtmlParser:
+        """
+        Returns a deep-copy independent instance of the parser
+        """
+        return HtmlParser(
+            self._html,
+            self._url,
+            clean_settings=self._clean_settings,
+            link_settings=self._link_settings,
+        )
+
+    @property
+    def current(self) -> LexborHTMLParser:
+        """
+        Direct access to the underlying active LexborHTMLParser instance
+        """
+        return self._current
+
+    def _html_text(self) -> str | None:
+        parsed_html = parse_string(self._current.html)
+        return parsed_html.value or None
+
+    def _has_content(self) -> bool:
+        html = self._html_text()
+        if not html:
+            return False
+        body = self._current.body
+        if body is None:
+            return bool(html)
+        text = safe_string(body.text(separator=" ", strip=True))
+        return bool(text)
+
+    def get_clean_content(self) -> str:
+        """
+        Clean the page with tiered fallbacks and return non-empty HTML.
+
+        Tries user/effective settings first (ideal → no-scope → emergency).
+        If those yield no content and settings were customized, falls back to
+        built-in defaults and logs an error. Raises if every attempt fails.
+        """
+        content = self._clean_with_settings(self._clean_settings)
+        if content is not None:
+            return content
+
+        if self._clean_settings.is_customized():
+            fallback = self._clean_with_settings(CleanSettings.defaults())
+            if fallback is not None:
+                self.error(
+                    f"User clean settings for {self._url!r} yielded no content; "
+                    f"fell back to defaults ({len(fallback)} chars remaining)"
+                )
+                return fallback
+
+        raise InternalCoreException(
+            f"Failed to extract clean content from {self._url!r}",
+            retryable=Retryability.NON_RETRYABLE,
+            error="all cleaning tiers produced empty content",
+        )
+
+    def _clean_with_settings(self, settings: CleanSettings) -> str | None:
+        for tier in ("ideal", "fallback", "emergency"):
+            self.reset()
+            self._apply_tier(settings, tier)
+            if self._has_content():
+                html = self._html_text()
+                if html is not None:
+                    return html
+            self.warning(f"[HtmlParser] Clean tier {tier!r} produced no content for {self._url!r}")
+        return None
+
+    def _apply_tier(self, settings: CleanSettings, tier: str) -> None:
+        if tier == "ideal":
+            self.prune_noise(settings, emergency=False)
+            self.apply_remove_selectors(settings)
+            self.scope_to_content_root(settings)
+            self.sanitize_attributes(settings)
+        elif tier == "fallback":
+            self.prune_noise(settings, emergency=False)
+            self.apply_remove_selectors(settings)
+            self.sanitize_attributes(settings)
+        else:
+            self.prune_noise(settings, emergency=True)
+            self.apply_remove_selectors(settings)
+            self.sanitize_attributes(settings)
+
+    def get_metadata(self) -> PageMetadata:
+        """
+        Extracts and validates page metadata into a structured model
+        """
+        title_node = self._current.css_first("title")
+        page_title = title_node.text(strip=True) if title_node else ""
+
+        base_node = self._current.css_first("base")
+        base_url = base_node.attributes.get("href") if base_node and base_node.attributes else None
+
+        meta_tags: dict[str, str] = {}
+        for m in iterator(self._current.css("meta"), where=lambda node: bool(node.attributes)):
+            attrs = safe_dict(m.attributes)
+            key = attrs.get("name") or attrs.get("property")
+            content = attrs.get("content")
+            if key and content:
+                meta_tags[key] = content
+
+        return PageMetadata(
+            title=page_title,
+            base_url=base_url,
+            meta=meta_tags,
+        )
+
+    def _base_for_links(self) -> str:
+        metadata = self.get_metadata()
+        if metadata.base_url:
+            return urljoin(self._url, metadata.base_url)
+        return self._url
+
+    def resolve_href(self, href: str, *, base: str | None = None) -> str | None:
+        """
+        Resolve a raw href to an absolute http(s) URL, or None if it should be skipped.
+
+        Fragments are always stripped. Query strings are kept unless
+        ``link_settings.should_strip_query`` says otherwise for the URL.
+        """
+        raw = href.strip()
+        if not raw or raw.startswith("#"):
+            return None
+
+        absolute = urljoin(base or self._base_for_links(), raw)
+        absolute, _frag = urldefrag(absolute)
+        parsed = urlparse(absolute)
+        scheme = parsed.scheme.lower()
+        if scheme in _SKIP_SCHEMES or scheme not in {"http", "https"}:
+            return None
+        if self._link_settings and self._link_settings.should_strip_query(absolute) and parsed.query:
+            absolute = urlunparse(parsed._replace(query=""))
+        return absolute
+
+    def get_links(self) -> Iterator[LinkItem]:
+        """
+        Extracts links as absolute URLs with anchor text.
+        """
+        base = self._base_for_links()
+
+        def has_href(node: Any) -> bool:
+            return bool(safe_dict(node.attributes).get("href"))
+
+        for a in iterator(self._current.css("a[href]"), where=has_href):
+            resolved = self.resolve_href(a.attributes["href"], base=base)
+            if not resolved:
+                continue
+            yield LinkItem(href=resolved, text=parse_string(a.text(strip=True)).value)
+
+    def find_content_root(self, settings: CleanSettings | None = None) -> Any:
+        """
+        Navigates to and returns the optimal starting content container node
+        """
+        effective = settings or self._clean_settings
+        selector = effective.effective_content_selector(self._url)
+        if selector:
+            if node := self._current.css_first(selector):
+                return node
+        return self._current.css_first("body")
+
+    def prune_noise(self, settings: CleanSettings | None = None, *, emergency: bool = False) -> None:
+        """
+        Removes unhelpful structural elements while optionally preserving search UI
+        """
+        effective = settings or self._clean_settings
+        for tag in effective.effective_noise_tags(emergency=emergency):
+            for node in self._current.css(tag):
+                node.remove()
+
+        if emergency or not effective.preserve_header_nav_with_search:
+            return
+
+        for container in iterator(
+            self._current.css("header, nav"),
+            where=lambda _node: not _node.css_first(effective.search_selectors),
+        ):
+            container.remove()
+
+    def apply_remove_selectors(self, settings: CleanSettings | None = None) -> None:
+        """
+        Remove nodes matching user-provided CSS selectors.
+        """
+        effective = settings or self._clean_settings
+        for selector in effective.remove_selectors:
+            for node in self._current.css(selector):
+                node.remove()
+
+    def sanitize_attributes(self, settings: CleanSettings | None = None) -> None:
+        """
+        Strips non-allowed attributes across the tree
+        """
+        effective = settings or self._clean_settings
+        allowed = effective.effective_allowed_attributes()
+        for node in iterator(self._current.css("*"), where=lambda n: bool(n.attributes)):
+            for key in iterator(
+                list(node.attrs.keys()),
+                where=lambda attr: attr.lower() not in allowed,
+            ):
+                del node.attrs[key]
+
+    def scope_to_content_root(self, settings: CleanSettings | None = None) -> None:
+        """
+        Replace the working tree with the best content container, when found.
+        """
+        effective = settings or self._clean_settings
+        root = self.find_content_root(effective)
+        if root is None:
+            self.warning(f"[HtmlParser] No content root found for {self._url!r}")
+            return
+
+        html = safe_string(root.html)
+        if not html:
+            self.warning(f"[HtmlParser] Content root for {self._url!r} had empty HTML")
+            return
+
+        text = safe_string(root.text(separator=" ", strip=True))
+        if not text:
+            self.warning(f"[HtmlParser] Content root for {self._url!r} had no text content")
+            return
+
+        self._current = LexborHTMLParser(html)
+
+    def clean(self, settings: CleanSettings | None = None) -> None:
+        """
+        Ideal-path clean: prune, remove selectors, scope, sanitize.
+
+        Prefer ``get_clean_content`` when you need tiered fallbacks.
+        """
+        effective = settings or self._clean_settings
+        self._apply_tier(effective, "ideal")

corekit/html/scraper.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+import asyncio
+from http import HTTPMethod
+from typing import Any, AsyncIterator, cast
+
+import httpx
+
+from corekit.html.client import WebscrapeClient
+from corekit.html.models import Page
+from corekit.html.parser import HtmlParser
+from corekit.html.settings import CleanSettings, LinkFilterSettings
+from corekit.html.state import ScraperState
+from corekit.http import BaseHttpResponse
+from corekit.observability import Benchmarkable
+from corekit.utils import iterator
+
+
+class Scraper(Benchmarkable):
+    def __init__(
+        self,
+        start_urls: list[str],
+        settings: LinkFilterSettings,
+        *,
+        clean_settings: CleanSettings | None = None,
+        concurrency: int = 10,
+        request_timeout: float = 10.0,
+        retries: int = 2,
+    ) -> None:
+        super().__init__()
+        self.concurrency = concurrency
+        self.semaphore = asyncio.Semaphore(concurrency)
+        self.timeout = request_timeout
+        self.retries = retries
+        self.settings = settings
+        self.clean_settings = clean_settings or CleanSettings.defaults()
+
+        self._limits = httpx.Limits(
+            max_keepalive_connections=concurrency,
+            max_connections=concurrency * 2,
+        )
+        self._http: httpx.AsyncClient | None = None
+        self._client: WebscrapeClient | None = None
+        self._state = ScraperState(start_urls)
+
+    def initialize(self) -> None:
+        self._http = httpx.AsyncClient(
+            limits=self._limits,
+            timeout=self.timeout,
+            follow_redirects=True,
+        )
+        self._client = WebscrapeClient(
+            http_client=self._http,
+            timeout=self.timeout,
+            retries=self.retries,
+            base_delay=0.5,
+        )
+
+    async def close(self) -> None:
+        if self._http is not None:
+            await self._http.aclose()
+            self._http = None
+            self._client = None
+
+    async def __aenter__(self) -> Scraper:
+        self.initialize()
+        return self
+
+    async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        await self.close()
+
+    async def _fetch(self, url: str) -> BaseHttpResponse:
+        if self._client is None:
+            raise RuntimeError("Scraper must be initialized before running.")
+        response = await self._client.request(HTTPMethod.GET, url)
+        response.raise_for_status()
+        return response
+
+    async def _worker(self, worker_id: int) -> None:
+        while True:
+            url = await self._state.get_next_url()
+            try:
+                async with self.semaphore:
+                    self.info(f"[Worker {worker_id}] Fetching: {url}")
+                    response = await self._fetch(url)
+
+                page_url = response.url or url
+                tree = HtmlParser(
+                    response.text,
+                    page_url,
+                    clean_settings=self.clean_settings,
+                    link_settings=self.settings,
+                )
+                for link in iterator(tree.get_links(), where=lambda item: self.settings.should_visit(item.href)):
+                    await self._state.enqueue(link.href)
+
+                if self.settings.should_scrape(url) or self.settings.should_scrape(page_url):
+                    page = Page(
+                        metadata=tree.get_metadata(),
+                        content=tree.get_clean_content(),
+                    )
+                    await self._state.record_output(page)
+
+            except Exception as e:
+                self.error(f"[Worker {worker_id}] Error on {url}: {e}")
+
+            finally:
+                self._state.task_done()
+
+    async def run(self) -> AsyncIterator[Page]:
+        if not self._client:
+            raise RuntimeError("Scraper must be initialized before running.")
+
+        workers = [asyncio.create_task(self._worker(i)) for i in range(self.concurrency)]
+        monitor_task = asyncio.create_task(self._state.join())
+
+        try:
+            while True:
+                result = await self._state.get_output()
+                if result is ScraperState.FINISHED_TOKEN:
+                    break
+                yield cast(Page, result)
+                self._state.output_task_done()
+        finally:
+            monitor_task.cancel()
+            for w in workers:
+                w.cancel()

corekit/html/settings.py

@@ -0,0 +1,197 @@
+"""
+Filter and cleaning policy for crawling and HTML extraction.
+"""
+
+from __future__ import annotations
+
+import itertools
+import re
+from dataclasses import dataclass, field
+
+OptionalPattern = re.Pattern[str] | str | None
+
+DEFAULT_ALLOWED_ATTRIBUTES: frozenset[str] = frozenset(
+    {
+        "id",
+        "class",
+        "name",
+        "type",
+        "href",
+        "src",
+        "role",
+        "placeholder",
+        "value",
+        "data-id",
+        "data-testid",
+    }
+)
+DEFAULT_NOISE_TAGS: frozenset[str] = frozenset({"script", "style", "noscript", "iframe", "footer", "aside"})
+EMERGENCY_NOISE_TAGS: frozenset[str] = frozenset({"script", "style", "noscript"})
+DEFAULT_CONTENT_SELECTORS: tuple[str, ...] = (
+    "main",
+    "article",
+    "[role='main']",
+    "#content",
+    ".post-content",
+    ".entry-content",
+    ".article-body",
+)
+DEFAULT_SEARCH_SELECTORS: str = "form[role='search'], input[type='search'], [id*='search' i], [class*='search' i]"
+
+
+def compile_pattern(pattern: OptionalPattern) -> re.Pattern[str] | None:
+    if not pattern:
+        return None
+    if isinstance(pattern, str):
+        return re.compile(pattern)
+    return pattern
+
+
+def compile_patterns(patterns: list[OptionalPattern] | None) -> list[re.Pattern[str]]:
+    if not patterns:
+        return []
+    return [compiled for p in patterns if (compiled := compile_pattern(p)) is not None]
+
+
+class LinkFilterSettings:
+    scrape: list[re.Pattern[str]]
+    visit_only: list[re.Pattern[str]]
+    blocked: list[re.Pattern[str]]
+    strip_query: bool | list[re.Pattern[str]]
+
+    def __init__(
+        self,
+        scrape: list[OptionalPattern] | None = None,
+        visit_only: list[OptionalPattern] | None = None,
+        blocked: list[OptionalPattern] | None = None,
+        strip_query: bool | list[OptionalPattern] = False,
+    ) -> None:
+        self.scrape = compile_patterns(scrape)
+        self.visit_only = compile_patterns(visit_only)
+        self.blocked = compile_patterns(blocked)
+        if isinstance(strip_query, bool):
+            self.strip_query = strip_query
+        else:
+            self.strip_query = compile_patterns(strip_query)
+
+    def is_blocked(self, url: str) -> bool:
+        """
+        Returns True if the URL matches any of the patterns in self.blocked
+        """
+        return any(p.search(url) for p in self.blocked)
+
+    def should_scrape(self, url: str) -> bool:
+        """
+        Returns True if the URL matches any scrape pattern and is not blocked.
+
+        Empty scrape patterns mean nothing is scraped (allowlist-first).
+        """
+        return not self.is_blocked(url) and any(p.search(url) for p in self.scrape)
+
+    def should_visit(self, url: str) -> bool:
+        """
+        Returns True if the URL matches scrape or visit_only and is not blocked.
+
+        Empty patterns mean discovered links are not enqueued (start URLs only).
+        """
+        return not self.is_blocked(url) and any(p.search(url) for p in itertools.chain(self.scrape, self.visit_only))
+
+    def should_strip_query(self, url: str) -> bool:
+        """
+        Whether query params should be dropped for this URL.
+
+        ``False`` keeps them (default). ``True`` strips always. A pattern list
+        strips only when the URL matches.
+        """
+        if self.strip_query is False:
+            return False
+        if self.strip_query is True:
+            return True
+        return any(p.search(url) for p in self.strip_query)
+
+
+@dataclass(slots=True)
+class CleanSettings:
+    """
+    Page cleaning policy.
+
+    When fields are left unset, built-in defaults apply. Explicit user values
+    win on conflict (e.g. a custom allowlist replaces the default attribute
+    set; ``keep_tags`` prevents default noise removal).
+    """
+
+    content_selectors: list[str] | None = None
+    url_content_selectors: list[tuple[OptionalPattern, str]] = field(default_factory=list)
+    remove_selectors: list[str] = field(default_factory=list)
+    noise_tags: list[str] | None = None
+    extra_noise_tags: list[str] = field(default_factory=list)
+    keep_tags: list[str] = field(default_factory=list)
+    allowed_attributes: set[str] | None = None
+    extra_allowed_attributes: set[str] = field(default_factory=set)
+    disallowed_attributes: set[str] = field(default_factory=set)
+    preserve_header_nav_with_search: bool = True
+    search_selectors: str = DEFAULT_SEARCH_SELECTORS
+
+    @classmethod
+    def defaults(cls) -> CleanSettings:
+        """
+        Built-in fallback policy used when user settings yield no content.
+        """
+        return cls()
+
+    def is_customized(self) -> bool:
+        """
+        True when any field differs from an empty/default construction.
+        """
+        baseline = CleanSettings.defaults()
+        return (
+            self.content_selectors is not None
+            or bool(self.url_content_selectors)
+            or bool(self.remove_selectors)
+            or self.noise_tags is not None
+            or bool(self.extra_noise_tags)
+            or bool(self.keep_tags)
+            or self.allowed_attributes is not None
+            or bool(self.extra_allowed_attributes)
+            or bool(self.disallowed_attributes)
+            or self.preserve_header_nav_with_search != baseline.preserve_header_nav_with_search
+            or self.search_selectors != baseline.search_selectors
+        )
+
+    def effective_noise_tags(self, *, emergency: bool = False) -> set[str]:
+        if emergency:
+            tags = set(EMERGENCY_NOISE_TAGS)
+        elif self.noise_tags is not None:
+            tags = {t.lower() for t in self.noise_tags}
+        else:
+            tags = {t.lower() for t in DEFAULT_NOISE_TAGS}
+            tags.update(t.lower() for t in self.extra_noise_tags)
+        tags.difference_update(t.lower() for t in self.keep_tags)
+        return tags
+
+    def effective_allowed_attributes(self) -> set[str]:
+        if self.allowed_attributes is not None:
+            allowed = {a.lower() for a in self.allowed_attributes}
+        else:
+            allowed = {a.lower() for a in DEFAULT_ALLOWED_ATTRIBUTES}
+            allowed.update(a.lower() for a in self.extra_allowed_attributes)
+        allowed.difference_update(a.lower() for a in self.disallowed_attributes)
+        return allowed
+
+    def effective_content_selector(self, url: str) -> str:
+        """
+        Comma-joined CSS selector list for this URL.
+
+        URL-specific selectors are prepended so they win over global/default ones.
+        """
+        matched: list[str] = []
+        for pattern, selector in self.url_content_selectors:
+            compiled = compile_pattern(pattern)
+            if compiled and compiled.search(url):
+                matched.append(selector)
+        if self.content_selectors is not None:
+            base = list(self.content_selectors)
+        else:
+            base = list(DEFAULT_CONTENT_SELECTORS)
+        combined = matched + base
+        return ", ".join(s for s in combined if s)