python-corekit 0.4.1__tar.gz → 0.5.0__tar.gz

{python_corekit-0.4.1/python_corekit.egg-info → python_corekit-0.5.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: python-corekit
-Version: 0.4.1
-Summary: Shared foundations for Python projects: logging, benchmarking, registries, FastAPI application and routers, SQL statements and migrations, background tasks, ETL, and LLM chat/tool primitives
+Version: 0.5.0
+Summary: Shared foundations for Python projects: logging, benchmarking, registries, FastAPI application and routers, SQL statements and migrations, background tasks, ETL, LLM chat/tool primitives, and HTML scraping
 Author: Steven Jacobsen
 License-Expression: MIT
 Project-URL: Homepage, https://github.com/stevejaker/corekit
@@ -23,6 +23,7 @@ Requires-Dist: sqlmodel<0.1,>=0.0.16
 Requires-Dist: SQLAlchemy<3,>=2.0
 Requires-Dist: redis<7,>=5.0
 Requires-Dist: httpx<1,>=0.27
+Requires-Dist: selectolax<1,>=0.3
 Requires-Dist: docker<8,>=7.0
 Requires-Dist: PyYAML<7,>=6.0
 Requires-Dist: dill<0.5,>=0.3.8
@@ -347,13 +348,13 @@ class GithubClient(BaseApiClient):
         return "https://api.github.com"
 
 response = GithubClient().get("/users/octocat")
-response.data["login"]
+response.data.login
 ```
 
-Retries 429 and 5xx with exponential backoff. Every response is a
-`BaseApiResponse`, so a non-JSON error page leaves `data` empty rather than
-raising. The same methods are awaitable inside a running event loop:
-`await client.get(...)`.
+Retries 429 and 5xx with exponential backoff. ``BaseHttpClient`` returns a
+``BaseHttpResponse`` (text + metadata); ``BaseApiClient`` adds structured
+``data`` from JSON (``None`` when the body is not JSON). The same methods are
+awaitable inside a running event loop: `await client.get(...)`.
 
 ## Serialization
 

{python_corekit-0.4.1 → python_corekit-0.5.0}/README.md

@@ -311,13 +311,13 @@ class GithubClient(BaseApiClient):
         return "https://api.github.com"
 
 response = GithubClient().get("/users/octocat")
-response.data["login"]
+response.data.login
 ```
 
-Retries 429 and 5xx with exponential backoff. Every response is a
-`BaseApiResponse`, so a non-JSON error page leaves `data` empty rather than
-raising. The same methods are awaitable inside a running event loop:
-`await client.get(...)`.
+Retries 429 and 5xx with exponential backoff. ``BaseHttpClient`` returns a
+``BaseHttpResponse`` (text + metadata); ``BaseApiClient`` adds structured
+``data`` from JSON (``None`` when the body is not JSON). The same methods are
+awaitable inside a running event loop: `await client.get(...)`.
 
 ## Serialization
 

{python_corekit-0.4.1 → python_corekit-0.5.0}/corekit/connections/sql/connection.py

@@ -330,15 +330,19 @@ class SQLConnection(Connectable):
         """
         return list(self._exec(query.build()))
 
-    def delete_many_by_query(self, query: Query) -> None:
+    def delete_many_by_query(self, query: Query) -> int:
         """
         Delete every row matching a Query.
+
+        Returns the number of rows deleted.
         """
         self.require_writable()
         self._start_session()
-        for item in self.fetch_many_by_query(query):
+        items = self.fetch_many_by_query(query)
+        for item in items:
             self.session.delete(item)
         self.session.commit()
+        return len(items)
 
     def execute(self, operation: Operation) -> Any:
         """

python_corekit-0.5.0/corekit/html/__init__.py

@@ -0,0 +1,22 @@
+"""
+HTML parsing and crawling helpers.
+"""
+
+from corekit.html.client import WebscrapeClient
+from corekit.html.models import LinkItem, Page, PageMetadata
+from corekit.html.parser import HtmlParser
+from corekit.html.scraper import Scraper
+from corekit.html.settings import CleanSettings, LinkFilterSettings
+from corekit.html.state import ScraperState
+
+__all__ = [
+    "CleanSettings",
+    "HtmlParser",
+    "LinkFilterSettings",
+    "LinkItem",
+    "Page",
+    "PageMetadata",
+    "Scraper",
+    "ScraperState",
+    "WebscrapeClient",
+]

python_corekit-0.5.0/corekit/html/client.py

@@ -0,0 +1,22 @@
+from typing import Any
+
+from corekit.http import BaseHttpClient
+
+
+class WebscrapeClient(BaseHttpClient):
+    """
+    HTTP client with browser-like default headers for scraping.
+
+    Inherits GET-only ``supported_methods`` from ``BaseHttpClient``.
+    """
+
+    def __init__(self, **kwargs: Any) -> None:
+        kwargs.setdefault("strict", False)
+        super().__init__(**kwargs)
+
+    @property
+    def headers(self) -> dict[str, str]:
+        return {
+            "Accept": "text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:156.0) Gecko/20100101 Firefox/156.0",
+        }

python_corekit-0.5.0/corekit/html/models.py

@@ -0,0 +1,20 @@
+from dataclasses import dataclass, field
+
+
+@dataclass(slots=True)
+class PageMetadata:
+    title: str = ""
+    base_url: str | None = None
+    meta: dict[str, str] = field(default_factory=dict)
+
+
+@dataclass(slots=True)
+class LinkItem:
+    href: str
+    text: str | None = None
+
+
+@dataclass(slots=True)
+class Page:
+    metadata: PageMetadata
+    content: str

python_corekit-0.5.0/corekit/html/parser.py

@@ -0,0 +1,272 @@
+from __future__ import annotations
+
+from typing import Any, Iterator
+from urllib.parse import urldefrag, urljoin, urlparse, urlunparse
+
+from selectolax.lexbor import LexborHTMLParser
+
+from corekit.exceptions import InternalCoreException
+from corekit.exceptions.enum import Retryability
+from corekit.html.models import LinkItem, PageMetadata
+from corekit.html.settings import CleanSettings, LinkFilterSettings
+from corekit.observability import Loggable
+from corekit.utils import iterator, parse_string, safe_dict, safe_string
+
+_SKIP_SCHEMES: frozenset[str] = frozenset({"", "javascript", "mailto", "tel", "data", "blob", "about"})
+
+
+class HtmlParser(Loggable):
+    def __init__(
+        self,
+        html: str | bytes,
+        url: str,
+        *,
+        clean_settings: CleanSettings | None = None,
+        link_settings: LinkFilterSettings | None = None,
+    ) -> None:
+        super().__init__()
+        self._html = html
+        self._url = url
+        self._clean_settings = clean_settings or CleanSettings.defaults()
+        self._link_settings = link_settings
+        self._original = LexborHTMLParser(html)
+        self._current = LexborHTMLParser(html)
+
+    def reset(self) -> None:
+        """
+        Resets the current working tree back to the pristine original state
+        """
+        self._current = LexborHTMLParser(self._html)
+
+    def clone(self) -> HtmlParser:
+        """
+        Returns a deep-copy independent instance of the parser
+        """
+        return HtmlParser(
+            self._html,
+            self._url,
+            clean_settings=self._clean_settings,
+            link_settings=self._link_settings,
+        )
+
+    @property
+    def current(self) -> LexborHTMLParser:
+        """
+        Direct access to the underlying active LexborHTMLParser instance
+        """
+        return self._current
+
+    def _html_text(self) -> str | None:
+        parsed_html = parse_string(self._current.html)
+        return parsed_html.value or None
+
+    def _has_content(self) -> bool:
+        html = self._html_text()
+        if not html:
+            return False
+        body = self._current.body
+        if body is None:
+            return bool(html)
+        text = safe_string(body.text(separator=" ", strip=True))
+        return bool(text)
+
+    def get_clean_content(self) -> str:
+        """
+        Clean the page with tiered fallbacks and return non-empty HTML.
+
+        Tries user/effective settings first (ideal → no-scope → emergency).
+        If those yield no content and settings were customized, falls back to
+        built-in defaults and logs an error. Raises if every attempt fails.
+        """
+        content = self._clean_with_settings(self._clean_settings)
+        if content is not None:
+            return content
+
+        if self._clean_settings.is_customized():
+            fallback = self._clean_with_settings(CleanSettings.defaults())
+            if fallback is not None:
+                self.error(
+                    f"User clean settings for {self._url!r} yielded no content; "
+                    f"fell back to defaults ({len(fallback)} chars remaining)"
+                )
+                return fallback
+
+        raise InternalCoreException(
+            f"Failed to extract clean content from {self._url!r}",
+            retryable=Retryability.NON_RETRYABLE,
+            error="all cleaning tiers produced empty content",
+        )
+
+    def _clean_with_settings(self, settings: CleanSettings) -> str | None:
+        for tier in ("ideal", "fallback", "emergency"):
+            self.reset()
+            self._apply_tier(settings, tier)
+            if self._has_content():
+                html = self._html_text()
+                if html is not None:
+                    return html
+            self.warning(f"[HtmlParser] Clean tier {tier!r} produced no content for {self._url!r}")
+        return None
+
+    def _apply_tier(self, settings: CleanSettings, tier: str) -> None:
+        if tier == "ideal":
+            self.prune_noise(settings, emergency=False)
+            self.apply_remove_selectors(settings)
+            self.scope_to_content_root(settings)
+            self.sanitize_attributes(settings)
+        elif tier == "fallback":
+            self.prune_noise(settings, emergency=False)
+            self.apply_remove_selectors(settings)
+            self.sanitize_attributes(settings)
+        else:
+            self.prune_noise(settings, emergency=True)
+            self.apply_remove_selectors(settings)
+            self.sanitize_attributes(settings)
+
+    def get_metadata(self) -> PageMetadata:
+        """
+        Extracts and validates page metadata into a structured model
+        """
+        title_node = self._current.css_first("title")
+        page_title = title_node.text(strip=True) if title_node else ""
+
+        base_node = self._current.css_first("base")
+        base_url = base_node.attributes.get("href") if base_node and base_node.attributes else None
+
+        meta_tags: dict[str, str] = {}
+        for m in iterator(self._current.css("meta"), where=lambda node: bool(node.attributes)):
+            attrs = safe_dict(m.attributes)
+            key = attrs.get("name") or attrs.get("property")
+            content = attrs.get("content")
+            if key and content:
+                meta_tags[key] = content
+
+        return PageMetadata(
+            title=page_title,
+            base_url=base_url,
+            meta=meta_tags,
+        )
+
+    def _base_for_links(self) -> str:
+        metadata = self.get_metadata()
+        if metadata.base_url:
+            return urljoin(self._url, metadata.base_url)
+        return self._url
+
+    def resolve_href(self, href: str, *, base: str | None = None) -> str | None:
+        """
+        Resolve a raw href to an absolute http(s) URL, or None if it should be skipped.
+
+        Fragments are always stripped. Query strings are kept unless
+        ``link_settings.should_strip_query`` says otherwise for the URL.
+        """
+        raw = href.strip()
+        if not raw or raw.startswith("#"):
+            return None
+
+        absolute = urljoin(base or self._base_for_links(), raw)
+        absolute, _frag = urldefrag(absolute)
+        parsed = urlparse(absolute)
+        scheme = parsed.scheme.lower()
+        if scheme in _SKIP_SCHEMES or scheme not in {"http", "https"}:
+            return None
+        if self._link_settings and self._link_settings.should_strip_query(absolute) and parsed.query:
+            absolute = urlunparse(parsed._replace(query=""))
+        return absolute
+
+    def get_links(self) -> Iterator[LinkItem]:
+        """
+        Extracts links as absolute URLs with anchor text.
+        """
+        base = self._base_for_links()
+
+        def has_href(node: Any) -> bool:
+            return bool(safe_dict(node.attributes).get("href"))
+
+        for a in iterator(self._current.css("a[href]"), where=has_href):
+            resolved = self.resolve_href(a.attributes["href"], base=base)
+            if not resolved:
+                continue
+            yield LinkItem(href=resolved, text=parse_string(a.text(strip=True)).value)
+
+    def find_content_root(self, settings: CleanSettings | None = None) -> Any:
+        """
+        Navigates to and returns the optimal starting content container node
+        """
+        effective = settings or self._clean_settings
+        selector = effective.effective_content_selector(self._url)
+        if selector:
+            if node := self._current.css_first(selector):
+                return node
+        return self._current.css_first("body")
+
+    def prune_noise(self, settings: CleanSettings | None = None, *, emergency: bool = False) -> None:
+        """
+        Removes unhelpful structural elements while optionally preserving search UI
+        """
+        effective = settings or self._clean_settings
+        for tag in effective.effective_noise_tags(emergency=emergency):
+            for node in self._current.css(tag):
+                node.remove()
+
+        if emergency or not effective.preserve_header_nav_with_search:
+            return
+
+        for container in iterator(
+            self._current.css("header, nav"),
+            where=lambda _node: not _node.css_first(effective.search_selectors),
+        ):
+            container.remove()
+
+    def apply_remove_selectors(self, settings: CleanSettings | None = None) -> None:
+        """
+        Remove nodes matching user-provided CSS selectors.
+        """
+        effective = settings or self._clean_settings
+        for selector in effective.remove_selectors:
+            for node in self._current.css(selector):
+                node.remove()
+
+    def sanitize_attributes(self, settings: CleanSettings | None = None) -> None:
+        """
+        Strips non-allowed attributes across the tree
+        """
+        effective = settings or self._clean_settings
+        allowed = effective.effective_allowed_attributes()
+        for node in iterator(self._current.css("*"), where=lambda n: bool(n.attributes)):
+            for key in iterator(
+                list(node.attrs.keys()),
+                where=lambda attr: attr.lower() not in allowed,
+            ):
+                del node.attrs[key]
+
+    def scope_to_content_root(self, settings: CleanSettings | None = None) -> None:
+        """
+        Replace the working tree with the best content container, when found.
+        """
+        effective = settings or self._clean_settings
+        root = self.find_content_root(effective)
+        if root is None:
+            self.warning(f"[HtmlParser] No content root found for {self._url!r}")
+            return
+
+        html = safe_string(root.html)
+        if not html:
+            self.warning(f"[HtmlParser] Content root for {self._url!r} had empty HTML")
+            return
+
+        text = safe_string(root.text(separator=" ", strip=True))
+        if not text:
+            self.warning(f"[HtmlParser] Content root for {self._url!r} had no text content")
+            return
+
+        self._current = LexborHTMLParser(html)
+
+    def clean(self, settings: CleanSettings | None = None) -> None:
+        """
+        Ideal-path clean: prune, remove selectors, scope, sanitize.
+
+        Prefer ``get_clean_content`` when you need tiered fallbacks.
+        """
+        effective = settings or self._clean_settings
+        self._apply_tier(effective, "ideal")

python_corekit-0.5.0/corekit/html/scraper.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+import asyncio
+from http import HTTPMethod
+from typing import Any, AsyncIterator, cast
+
+import httpx
+
+from corekit.html.client import WebscrapeClient
+from corekit.html.models import Page
+from corekit.html.parser import HtmlParser
+from corekit.html.settings import CleanSettings, LinkFilterSettings
+from corekit.html.state import ScraperState
+from corekit.http import BaseHttpResponse
+from corekit.observability import Benchmarkable
+from corekit.utils import iterator
+
+
+class Scraper(Benchmarkable):
+    def __init__(
+        self,
+        start_urls: list[str],
+        settings: LinkFilterSettings,
+        *,
+        clean_settings: CleanSettings | None = None,
+        concurrency: int = 10,
+        request_timeout: float = 10.0,
+        retries: int = 2,
+    ) -> None:
+        super().__init__()
+        self.concurrency = concurrency
+        self.semaphore = asyncio.Semaphore(concurrency)
+        self.timeout = request_timeout
+        self.retries = retries
+        self.settings = settings
+        self.clean_settings = clean_settings or CleanSettings.defaults()
+
+        self._limits = httpx.Limits(
+            max_keepalive_connections=concurrency,
+            max_connections=concurrency * 2,
+        )
+        self._http: httpx.AsyncClient | None = None
+        self._client: WebscrapeClient | None = None
+        self._state = ScraperState(start_urls)
+
+    def initialize(self) -> None:
+        self._http = httpx.AsyncClient(
+            limits=self._limits,
+            timeout=self.timeout,
+            follow_redirects=True,
+        )
+        self._client = WebscrapeClient(
+            http_client=self._http,
+            timeout=self.timeout,
+            retries=self.retries,
+            base_delay=0.5,
+        )
+
+    async def close(self) -> None:
+        if self._http is not None:
+            await self._http.aclose()
+            self._http = None
+            self._client = None
+
+    async def __aenter__(self) -> Scraper:
+        self.initialize()
+        return self
+
+    async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        await self.close()
+
+    async def _fetch(self, url: str) -> BaseHttpResponse:
+        if self._client is None:
+            raise RuntimeError("Scraper must be initialized before running.")
+        response = await self._client.request(HTTPMethod.GET, url)
+        response.raise_for_status()
+        return response
+
+    async def _worker(self, worker_id: int) -> None:
+        while True:
+            url = await self._state.get_next_url()
+            try:
+                async with self.semaphore:
+                    self.info(f"[Worker {worker_id}] Fetching: {url}")
+                    response = await self._fetch(url)
+
+                page_url = response.url or url
+                tree = HtmlParser(
+                    response.text,
+                    page_url,
+                    clean_settings=self.clean_settings,
+                    link_settings=self.settings,
+                )
+                for link in iterator(tree.get_links(), where=lambda item: self.settings.should_visit(item.href)):
+                    await self._state.enqueue(link.href)
+
+                if self.settings.should_scrape(url) or self.settings.should_scrape(page_url):
+                    page = Page(
+                        metadata=tree.get_metadata(),
+                        content=tree.get_clean_content(),
+                    )
+                    await self._state.record_output(page)
+
+            except Exception as e:
+                self.error(f"[Worker {worker_id}] Error on {url}: {e}")
+
+            finally:
+                self._state.task_done()
+
+    async def run(self) -> AsyncIterator[Page]:
+        if not self._client:
+            raise RuntimeError("Scraper must be initialized before running.")
+
+        workers = [asyncio.create_task(self._worker(i)) for i in range(self.concurrency)]
+        monitor_task = asyncio.create_task(self._state.join())
+
+        try:
+            while True:
+                result = await self._state.get_output()
+                if result is ScraperState.FINISHED_TOKEN:
+                    break
+                yield cast(Page, result)
+                self._state.output_task_done()
+        finally:
+            monitor_task.cancel()
+            for w in workers:
+                w.cancel()