wordlift-sdk 2.9.1__py3-none-any.whl → 2.10.2__py3-none-any.whl

wordlift_sdk/__init__.py

@@ -1,3 +1,3 @@
 from .main import run_kg_import_workflow
 
-__all__ = ['run_kg_import_workflow']
+__all__ = ["run_kg_import_workflow"]

wordlift_sdk/render/__init__.py

@@ -0,0 +1,30 @@
+"""Render and XHTML cleanup utilities."""
+
+from __future__ import annotations
+
+from .cleanup_options import CleanupOptions
+from .html_renderer import HtmlRenderer
+from .render_options import RenderOptions
+from .rendered_page import RenderedPage
+from .xhtml_cleaner import XhtmlCleaner
+
+
+def render_html(options: RenderOptions) -> RenderedPage:
+    """Wrapper for backward compatibility."""
+    return HtmlRenderer().render(options)
+
+
+def clean_xhtml(xhtml: str, options: CleanupOptions) -> str:
+    """Wrapper for backward compatibility."""
+    return XhtmlCleaner().clean(xhtml, options)
+
+
+__all__ = [
+    "CleanupOptions",
+    "HtmlRenderer",
+    "RenderOptions",
+    "RenderedPage",
+    "XhtmlCleaner",
+    "clean_xhtml",
+    "render_html",
+]

wordlift_sdk/render/browser.py

@@ -0,0 +1,132 @@
+"""Browser helper for rendering pages with Playwright."""
+
+from __future__ import annotations
+
+from contextlib import AbstractContextManager
+from dataclasses import dataclass
+from time import perf_counter
+
+try:
+    from playwright.sync_api import Error as PlaywrightError
+    from playwright.sync_api import sync_playwright
+except ImportError:  # pragma: no cover - runtime dependency
+    sync_playwright = None
+    PlaywrightError = Exception
+
+
+@dataclass
+class PageFetch:
+    response: object | None
+    elapsed_ms: float
+    resources: list[dict]
+
+
+class Browser(AbstractContextManager):
+    _DEFAULT_HEADERS = {
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.9",
+        "Referer": "https://wordlift.io",
+        "Upgrade-Insecure-Requests": "1",
+        "Sec-CH-UA": '"Not A(Brand";v="99", "Chromium";v="120", "Google Chrome";v="120"',
+        "Sec-CH-UA-Mobile": "?0",
+        "Sec-CH-UA-Platform": '"macOS"',
+    }
+
+    def __init__(
+        self,
+        *,
+        headless: bool,
+        timeout_ms: int,
+        wait_until: str,
+        locale: str | None = None,
+        user_agent: str | None = None,
+        viewport_width: int | None = None,
+        viewport_height: int | None = None,
+        ignore_https_errors: bool = False,
+    ) -> None:
+        self.headless = headless
+        self.timeout_ms = timeout_ms
+        self.wait_until = wait_until
+        self.locale = locale
+        self.user_agent = user_agent
+        self.viewport_width = viewport_width
+        self.viewport_height = viewport_height
+        self.ignore_https_errors = ignore_https_errors
+        self._playwright = None
+        self._browser = None
+        self._context = None
+
+    def __enter__(self) -> "Browser":
+        if sync_playwright is None:
+            raise RuntimeError(
+                "Playwright is not installed. Run: uv pip install playwright && playwright install"
+            )
+        self._playwright = sync_playwright().start()
+        self._browser = self._playwright.chromium.launch(headless=self.headless)
+        context_kwargs: dict[str, object] = {}
+        context_kwargs["locale"] = self.locale or "en-US"
+        context_kwargs["timezone_id"] = "America/New_York"
+        if self.user_agent:
+            context_kwargs["user_agent"] = self.user_agent
+        if self.viewport_width and self.viewport_height:
+            viewport = {"width": self.viewport_width, "height": self.viewport_height}
+        else:
+            viewport = {"width": 1365, "height": 768}
+        context_kwargs["viewport"] = viewport
+        context_kwargs["ignore_https_errors"] = self.ignore_https_errors
+        context_kwargs["extra_http_headers"] = dict(self._DEFAULT_HEADERS)
+        self._context = self._browser.new_context(**context_kwargs)
+        self._context.add_init_script(
+            """
+            Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+            Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
+            Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });
+            window.chrome = window.chrome || { runtime: {} };
+            const originalQuery = window.navigator.permissions.query;
+            window.navigator.permissions.query = (parameters) => (
+                parameters.name === 'notifications'
+                    ? Promise.resolve({ state: Notification.permission })
+                    : originalQuery(parameters)
+            );
+            """
+        )
+        return self
+
+    def __exit__(self, exc_type, exc, tb) -> None:
+        if self._context is not None:
+            self._context.close()
+        if self._browser is not None:
+            self._browser.close()
+        if self._playwright is not None:
+            self._playwright.stop()
+
+    def open(self, url: str) -> tuple[object | None, object | None, float, list[dict]]:
+        if self._context is None:
+            raise RuntimeError("Browser not initialized")
+        page = self._context.new_page()
+        resources: list[dict] = []
+
+        def handle_response(resp) -> None:
+            try:
+                request = resp.request
+                resources.append(
+                    {
+                        "url": resp.url,
+                        "status": resp.status,
+                        "resource_type": request.resource_type,
+                    }
+                )
+            except Exception:
+                return
+
+        page.on("response", handle_response)
+        start = perf_counter()
+        response = None
+        try:
+            response = page.goto(
+                url, wait_until=self.wait_until, timeout=self.timeout_ms
+            )
+        except PlaywrightError:
+            pass
+        elapsed_ms = (perf_counter() - start) * 1000
+        return page, response, elapsed_ms, resources

wordlift_sdk/render/cleanup_options.py

@@ -0,0 +1,24 @@
+"""Cleanup options for XHTML sanitization."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass
+class CleanupOptions:
+    max_xhtml_chars: int = 40000
+    max_text_node_chars: int = 400
+    remove_tags: tuple[str, ...] = (
+        "script",
+        "style",
+        "noscript",
+        "svg",
+        "canvas",
+        "iframe",
+        "form",
+        "input",
+        "button",
+        "nav",
+        "aside",
+    )

wordlift_sdk/render/html_renderer.py

@@ -0,0 +1,86 @@
+"""HTML rendering and XHTML conversion."""
+
+from __future__ import annotations
+
+import time
+from typing import Any
+from urllib.parse import urlparse
+
+from wordlift_sdk.utils import HtmlConverter
+
+from .browser import Browser
+
+from .render_options import RenderOptions
+from .rendered_page import RenderedPage
+
+
+class HtmlRenderer:
+    """Renders a web page using a browser and converts it to XHTML."""
+
+    def render(self, options: RenderOptions) -> RenderedPage:
+        """
+        Render a URL to HTML and XHTML.
+
+        Args:
+            options: Configuration for rendering (URL, headless, timeout, etc.).
+
+        Returns:
+            A RenderedPage object containing the HTML, XHTML, and status code.
+        """
+        ignore_https_errors = options.ignore_https_errors or self._is_localhost_url(
+            options.url
+        )
+        with Browser(
+            headless=options.headless,
+            timeout_ms=options.timeout_ms,
+            wait_until=options.wait_until,
+            locale=options.locale,
+            user_agent=options.user_agent,
+            viewport_width=options.viewport_width,
+            viewport_height=options.viewport_height,
+            ignore_https_errors=ignore_https_errors,
+        ) as browser:
+            page, response, _elapsed_ms, resources = browser.open(options.url)
+            if page is None:
+                raise RuntimeError("Failed to open page in browser.")
+            try:
+                html = self._safe_page_content(page, options.timeout_ms)
+            finally:
+                page.close()
+
+        xhtml = HtmlConverter().convert(html)
+
+        status_code = None
+        if response is not None:
+            try:
+                status_code = response.status
+            except Exception:
+                status_code = None
+        return RenderedPage(
+            html=html, xhtml=xhtml, status_code=status_code, resources=resources
+        )
+
+    def _is_localhost_url(self, url: str) -> bool:
+        try:
+            parsed = urlparse(url)
+        except Exception:
+            return False
+        host = (parsed.hostname or "").lower()
+        return host == "localhost" or host.endswith(".localhost")
+
+    def _safe_page_content(self, page: Any, timeout_ms: int, retries: int = 3) -> str:
+        for attempt in range(retries + 1):
+            try:
+                return page.content()
+            except Exception:
+                if attempt >= retries:
+                    raise
+                try:
+                    page.wait_for_load_state("networkidle", timeout=timeout_ms)
+                except Exception:
+                    try:
+                        page.wait_for_load_state("load", timeout=timeout_ms)
+                    except Exception:
+                        pass
+                time.sleep(0.2)
+        return page.content()

wordlift_sdk/render/render_options.py

@@ -0,0 +1,21 @@
+"""Render options for HTML to XHTML pipeline."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass
+class RenderOptions:
+    url: str
+    headless: bool = True
+    timeout_ms: int = 30000
+    wait_until: str = "networkidle"
+    locale: str = "en-US"
+    user_agent: str | None = (
+        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+        "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+    )
+    viewport_width: int = 1365
+    viewport_height: int = 768
+    ignore_https_errors: bool = False

wordlift_sdk/render/rendered_page.py

@@ -0,0 +1,13 @@
+"""Rendered page data returned by the renderer."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class RenderedPage:
+    html: str
+    xhtml: str
+    status_code: int | None = None
+    resources: list[dict] = field(default_factory=list)

wordlift_sdk/render/xhtml_cleaner.py

@@ -0,0 +1,126 @@
+"""XHTML cleanup utilities."""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+
+from .cleanup_options import CleanupOptions
+
+
+class XhtmlCleaner:
+    """Cleans and optimizes XHTML content."""
+
+    def clean(self, xhtml: str, options: CleanupOptions) -> str:
+        """
+        Clean an XHTML string based on the provided options.
+
+        Args:
+            xhtml: The XHTML string to clean.
+            options: Configuration for cleaning (tags to remove, max chars, etc.).
+
+        Returns:
+            The cleaned XHTML string.
+        """
+        try:
+            from lxml import html as lxml_html
+        except Exception as exc:
+            raise RuntimeError(
+                "lxml is required for XHTML cleanup. Install with: pip install lxml"
+            ) from exc
+        parser = lxml_html.HTMLParser(encoding="utf-8", recover=True)
+        doc = lxml_html.document_fromstring(xhtml, parser=parser)
+        self._strip_unwanted_tags(doc, options.remove_tags)
+        self._compact_text_nodes(doc, options.max_text_node_chars)
+        self._cap_text_content(doc, options.max_xhtml_chars)
+        cleaned = lxml_html.tostring(doc, encoding="unicode", method="xml")
+        if len(cleaned) > options.max_xhtml_chars:
+            self._trim_elements_to_size(doc, options.max_xhtml_chars)
+            cleaned = lxml_html.tostring(doc, encoding="unicode", method="xml")
+        return cleaned
+
+    def _strip_unwanted_tags(self, doc: Any, tags: tuple[str, ...]) -> None:
+        if not tags:
+            return
+        tag_expr = " | ".join(f"//{tag}" for tag in tags)
+        if not tag_expr:
+            return
+        for element in doc.xpath(tag_expr):
+            parent = element.getparent()
+            if parent is not None:
+                parent.remove(element)
+
+    def _compact_text(self, value: str | None, max_chars: int) -> str | None:
+        if value is None:
+            return None
+        text = re.sub(r"\s+", " ", value).strip()
+        if not text:
+            return None
+        if max_chars > 0 and len(text) > max_chars:
+            if max_chars <= 3:
+                text = text[:max_chars]
+            else:
+                text = text[: max_chars - 3].rstrip() + "..."
+        return text
+
+    def _compact_text_nodes(self, doc: Any, max_chars: int) -> None:
+        for element in doc.iter():
+            if hasattr(element, "text"):
+                element.text = self._compact_text(element.text, max_chars)
+            if hasattr(element, "tail"):
+                element.tail = self._compact_text(element.tail, max_chars)
+
+    def _cap_text_content(self, doc: Any, max_chars: int) -> None:
+        if max_chars <= 0:
+            return
+        remaining = max_chars
+        for element in doc.iter():
+            if hasattr(element, "text") and element.text:
+                if len(element.text) <= remaining:
+                    remaining -= len(element.text)
+                else:
+                    element.text = element.text[: max(0, remaining)].rstrip()
+                    remaining = 0
+            if remaining <= 0:
+                self._clear_text_after(doc, element)
+                break
+            if hasattr(element, "tail") and element.tail:
+                if len(element.tail) <= remaining:
+                    remaining -= len(element.tail)
+                else:
+                    element.tail = element.tail[: max(0, remaining)].rstrip()
+                    remaining = 0
+            if remaining <= 0:
+                self._clear_text_after(doc, element)
+                break
+
+    def _clear_text_after(self, doc: Any, stop_element: Any) -> None:
+        seen = False
+        for element in doc.iter():
+            if element is stop_element:
+                seen = True
+                continue
+            if not seen:
+                continue
+            if hasattr(element, "text") and element.text:
+                element.text = None
+            if hasattr(element, "tail") and element.tail:
+                element.tail = None
+
+    def _trim_elements_to_size(self, doc: Any, max_chars: int) -> None:
+        if max_chars <= 0:
+            return
+        try:
+            from lxml import html as lxml_html
+        except Exception:
+            return
+        elements = list(doc.iter())
+        for element in reversed(elements):
+            parent = element.getparent()
+            if parent is None:
+                continue
+            parent.remove(element)
+            current = lxml_html.tostring(doc, encoding="unicode", method="xml")
+            if len(current) <= max_chars:
+                return

wordlift_sdk/structured_data/__init__.py

@@ -0,0 +1,27 @@
+"""Structured data workflows and utilities."""
+
+from __future__ import annotations
+
+from .agent_generator import AgentGenerator
+from .dataset_resolver import DatasetResolver
+from .engine import StructuredDataOptions, StructuredDataResult
+from .schema_guide import SchemaGuide
+from .structured_data_engine import StructuredDataEngine
+from .yarrrml_pipeline import YarrrmlPipeline
+from .models import CreateRequest, GenerateRequest
+from .orchestrator import CreateWorkflow, GenerateWorkflow, resolve_api_key_from_context
+
+__all__ = [
+    "CreateRequest",
+    "CreateWorkflow",
+    "GenerateRequest",
+    "GenerateWorkflow",
+    "resolve_api_key_from_context",
+    "AgentGenerator",
+    "DatasetResolver",
+    "SchemaGuide",
+    "StructuredDataEngine",
+    "StructuredDataOptions",
+    "StructuredDataResult",
+    "YarrrmlPipeline",
+]

wordlift_sdk/structured_data/agent.py

@@ -0,0 +1,49 @@
+"""Agent-backed structured data generation."""
+
+from __future__ import annotations
+
+from typing import Callable
+from pathlib import Path
+
+from wordlift_sdk.structured_data.agent_generator import (
+    AgentGenerator as EngineAgentGenerator,
+)
+
+
+class AgentGenerator:
+    """Generates YARRRML and JSON-LD via the agent."""
+
+    def __init__(self, engine: EngineAgentGenerator | None = None) -> None:
+        self._engine = engine or EngineAgentGenerator()
+
+    def generate(
+        self,
+        url: str,
+        html: str,
+        xhtml: str,
+        cleaned_xhtml: str,
+        api_key: str,
+        dataset_uri: str,
+        target_type: str,
+        workdir: Path,
+        debug: bool,
+        max_retries: int,
+        max_nesting_depth: int,
+        quality_check: bool,
+        log: Callable[[str], None],
+    ) -> tuple[str, dict]:
+        return self._engine.generate_from_agent(
+            url,
+            html,
+            xhtml,
+            cleaned_xhtml,
+            api_key,
+            dataset_uri,
+            target_type,
+            workdir,
+            debug=debug,
+            max_retries=max_retries,
+            max_nesting_depth=max_nesting_depth,
+            quality_check=quality_check,
+            log=log,
+        )

wordlift_sdk/structured_data/agent_generator.py

@@ -0,0 +1,12 @@
+"""Agent-backed generation utilities."""
+
+from __future__ import annotations
+
+from .engine import generate_from_agent
+
+
+class AgentGenerator:
+    """Agent-backed generation utilities."""
+
+    def generate_from_agent(self, *args, **kwargs):
+        return generate_from_agent(*args, **kwargs)