wordlift-sdk 2.9.1__py3-none-any.whl → 2.10.2__py3-none-any.whl

wordlift_sdk/structured_data/inputs.py

@@ -0,0 +1,58 @@
+"""Input resolution helpers for structured data workflows."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from urllib.parse import urlparse
+
+
+def is_url(value: str) -> bool:
+    parsed = urlparse(value)
+    return parsed.scheme in {"http", "https"}
+
+
+def urls_from_sitemap(source: str) -> list[str]:
+    try:
+        import advertools as adv
+    except ImportError as exc:  # pragma: no cover - runtime dependency
+        raise RuntimeError(
+            "advertools is required. Install with: pip install advertools"
+        ) from exc
+    df = adv.sitemap_to_df(source)
+    if df is None or df.empty:
+        return []
+    for column in ("loc", "url"):
+        if column in df.columns:
+            values = df[column].dropna().astype(str).tolist()
+            return [value for value in values if value]
+    return df.iloc[:, 0].dropna().astype(str).tolist()
+
+
+def resolve_input_urls(value: str) -> list[str]:
+    path = Path(value)
+    if path.exists():
+        urls = urls_from_sitemap(str(path))
+        if not urls:
+            raise RuntimeError("No URLs found in sitemap file.")
+        return urls
+    if is_url(value):
+        try:
+            urls = urls_from_sitemap(value)
+            if urls:
+                return urls
+        except Exception:
+            pass
+        return [value]
+    raise RuntimeError("INPUT must be a sitemap URL/path or a page URL.")
+
+
+def filter_urls(urls: list[str], regex: str, max_pages: int | None) -> list[str]:
+    import re
+
+    pattern = re.compile(regex)
+    urls = [url for url in urls if pattern.search(url)]
+    if not urls:
+        raise RuntimeError("No URLs matched the provided regex.")
+    if max_pages is not None:
+        urls = urls[:max_pages]
+    return urls

wordlift_sdk/structured_data/io.py

@@ -0,0 +1,44 @@
+"""Output utilities for structured data workflows."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from rdflib import Graph
+
+
+_OUTPUT_FORMATS: dict[str, tuple[str, str]] = {
+    "ttl": ("turtle", "ttl"),
+    "jsonld": ("json-ld", "jsonld"),
+    "json-ld": ("json-ld", "jsonld"),
+    "rdf": ("xml", "rdf"),
+    "nt": ("nt", "nt"),
+    "nq": ("nquads", "nq"),
+}
+
+
+def write_output(path: Path, content: str) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(content)
+
+
+def default_output_paths(out_dir: Path, base_name: str) -> tuple[Path, Path]:
+    jsonld_path = out_dir / f"{base_name}.jsonld"
+    yarrml_path = out_dir / f"{base_name}.yarrml"
+    return jsonld_path, yarrml_path
+
+
+def normalize_output_format(value: str) -> tuple[str, str]:
+    key = value.strip().lower()
+    if key not in _OUTPUT_FORMATS:
+        supported = ", ".join(sorted({k for k in _OUTPUT_FORMATS if "-" not in k}))
+        raise RuntimeError(f"Unsupported format '{value}'. Choose from: {supported}.")
+    return _OUTPUT_FORMATS[key]
+
+
+def serialize_graph(graph: Graph, output_format: str) -> str:
+    rdflib_format, _ = normalize_output_format(output_format)
+    serialized = graph.serialize(format=rdflib_format)
+    if isinstance(serialized, bytes):
+        return serialized.decode("utf-8")
+    return serialized

wordlift_sdk/structured_data/materialization.py

@@ -0,0 +1,70 @@
+"""Materialization pipeline for YARRRML -> JSON-LD."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from wordlift_sdk.structured_data.yarrrml_pipeline import YarrrmlPipeline
+
+
+class MaterializationPipeline:
+    """Normalizes mappings, materializes JSON-LD, and post-processes output."""
+
+    def __init__(self, pipeline: YarrrmlPipeline | None = None) -> None:
+        self._pipeline = pipeline or YarrrmlPipeline()
+
+    def normalize(
+        self, yarrrml: str, url: str, xhtml_path: Path, target_type: str | None
+    ) -> tuple[str, list[dict]]:
+        return self._pipeline.normalize_mappings(
+            yarrrml, url, xhtml_path, target_type=target_type
+        )
+
+    def materialize(
+        self, normalized_yarrrml: str, xhtml_path: Path, workdir: Path, url: str
+    ) -> dict:
+        return self._pipeline.materialize_jsonld(
+            normalized_yarrrml, xhtml_path, workdir, url=url
+        )
+
+    def postprocess(
+        self,
+        jsonld_raw: dict,
+        mappings: list[dict],
+        cleaned_xhtml: str,
+        dataset_uri: str,
+        url: str,
+        target_type: str | None,
+    ) -> dict:
+        return self._pipeline.postprocess_jsonld(
+            jsonld_raw,
+            mappings,
+            cleaned_xhtml,
+            dataset_uri,
+            url,
+            target_type=target_type,
+        )
+
+    def run(
+        self,
+        yarrrml: str,
+        url: str,
+        cleaned_xhtml: str,
+        dataset_uri: str,
+        xhtml_path: Path,
+        workdir: Path,
+        target_type: str | None,
+    ) -> tuple[dict, list[dict]]:
+        normalized_yarrrml, mappings = self.normalize(
+            yarrrml, url, xhtml_path, target_type=target_type
+        )
+        jsonld_raw = self.materialize(normalized_yarrrml, xhtml_path, workdir, url=url)
+        jsonld = self.postprocess(
+            jsonld_raw,
+            mappings,
+            cleaned_xhtml,
+            dataset_uri,
+            url,
+            target_type=target_type,
+        )
+        return jsonld, mappings

wordlift_sdk/structured_data/models.py

@@ -0,0 +1,48 @@
+"""Request/response models for structured data workflows."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class CreateRequest:
+    url: str
+    target_type: str
+    output_dir: Path
+    base_name: str
+    jsonld_path: Path | None
+    yarrml_path: Path | None
+    api_key: str | None
+    base_url: str | None
+    debug: bool
+    headed: bool
+    timeout_ms: int
+    max_retries: int
+    quality_check: bool
+    max_xhtml_chars: int
+    max_text_node_chars: int
+    max_nesting_depth: int
+    verbose: bool
+    validate: bool
+    wait_until: str
+
+
+@dataclass
+class GenerateRequest:
+    input_value: str
+    yarrrml_path: Path
+    regex: str
+    output_dir: Path
+    output_format: str
+    concurrency: str
+    api_key: str | None
+    base_url: str | None
+    headed: bool
+    timeout_ms: int
+    wait_until: str
+    max_xhtml_chars: int
+    max_text_node_chars: int
+    max_pages: int | None
+    verbose: bool

wordlift_sdk/structured_data/orchestrator.py

@@ -0,0 +1,194 @@
+"""Orchestration for structured data workflows."""
+
+from __future__ import annotations
+
+import json
+from typing import Callable
+
+from wordlift_sdk.structured_data.engine import (
+    StructuredDataOptions,
+    StructuredDataResult,
+)
+from wordlift_sdk.structured_data.constants import DEFAULT_BASE_URL
+from wordlift_sdk.structured_data.structured_data_engine import StructuredDataEngine
+from wordlift_sdk.structured_data.yarrrml_pipeline import YarrrmlPipeline
+
+from .agent import AgentGenerator
+from .batch import BatchGenerator
+from .debug import echo_debug
+from .io import default_output_paths, write_output
+from .inputs import filter_urls, resolve_input_urls
+from .models import CreateRequest, GenerateRequest
+from .rendering import RenderPipeline
+from .validation import ValidationService
+
+
+class CreateWorkflow:
+    """Workflow for generating structured data from a single URL."""
+
+    def __init__(
+        self,
+        agent: AgentGenerator | None = None,
+        renderer: RenderPipeline | None = None,
+        validator: ValidationService | None = None,
+        engine: StructuredDataEngine | None = None,
+    ) -> None:
+        self._agent = agent or AgentGenerator()
+        self._renderer = renderer
+        self._validator = validator or ValidationService()
+        self._engine = engine or StructuredDataEngine()
+        self._yarrrml = YarrrmlPipeline()
+
+    def run(
+        self, request: CreateRequest, log: Callable[[str], None]
+    ) -> StructuredDataResult:
+        if not request.api_key:
+            raise RuntimeError(
+                "WORDLIFT_KEY is required (or set wordlift.api_key in config)."
+            )
+        base_url = request.base_url or DEFAULT_BASE_URL
+        dataset_uri = self._engine.get_dataset_uri(request.api_key, base_url=base_url)
+
+        renderer = self._renderer or RenderPipeline(
+            headed=request.headed,
+            timeout_ms=request.timeout_ms,
+            wait_until=request.wait_until,
+            max_xhtml_chars=request.max_xhtml_chars,
+            max_text_node_chars=request.max_text_node_chars,
+        )
+
+        rendered, cleaned_xhtml = renderer.render(request.url, log)
+
+        options = StructuredDataOptions(
+            url=request.url,
+            target_type=request.target_type,
+            dataset_uri=dataset_uri,
+            headless=not request.headed,
+            timeout_ms=request.timeout_ms,
+            wait_until=request.wait_until,
+            max_retries=request.max_retries,
+            max_xhtml_chars=request.max_xhtml_chars,
+            max_text_node_chars=request.max_text_node_chars,
+            max_nesting_depth=request.max_nesting_depth,
+            verbose=request.verbose,
+        )
+
+        workdir = request.output_dir / ".structured-data"
+        debug_path = workdir / "agent_debug.json"
+        try:
+            log("Generating YARRRML mapping and JSON-LD...")
+            yarrml, jsonld = self._agent.generate(
+                options.url,
+                rendered.html,
+                rendered.xhtml,
+                cleaned_xhtml,
+                request.api_key,
+                options.dataset_uri,
+                options.target_type,
+                workdir,
+                debug=request.debug,
+                max_retries=options.max_retries,
+                max_nesting_depth=options.max_nesting_depth,
+                quality_check=request.quality_check,
+                log=log,
+            )
+        except Exception:
+            if request.debug:
+                echo_debug(debug_path, log)
+            raise
+        if request.debug:
+            echo_debug(debug_path, log)
+
+        jsonld_path = request.jsonld_path
+        yarrml_path = request.yarrml_path
+        if jsonld_path is None or yarrml_path is None:
+            jsonld_path, yarrml_path = default_output_paths(
+                request.output_dir, request.base_name
+            )
+
+        write_output(jsonld_path, json.dumps(jsonld, indent=2))
+        yarrml = self._yarrrml.make_reusable_yarrrml(yarrml, request.url)
+        write_output(yarrml_path, yarrml)
+
+        if request.verbose:
+            mapping_validation_path = workdir / "mapping.validation.json"
+            if mapping_validation_path.exists():
+                try:
+                    validation_payload = json.loads(mapping_validation_path.read_text())
+                except Exception:
+                    validation_payload = {}
+                for warning in validation_payload.get("warnings", []):
+                    if "reviewRating dropped" in warning:
+                        log(warning)
+
+        if request.validate:
+            log("Validating JSON-LD output...")
+            report_text = self._validator.validate(
+                jsonld_path, request.target_type, workdir
+            )
+            log(report_text)
+
+        return StructuredDataResult(
+            jsonld=jsonld,
+            yarrml=yarrml,
+            jsonld_filename=str(jsonld_path),
+            yarrml_filename=str(yarrml_path),
+        )
+
+
+class GenerateWorkflow:
+    """Workflow for generating structured data in batch from YARRRML."""
+
+    def __init__(self, engine: StructuredDataEngine | None = None) -> None:
+        self._engine = engine or StructuredDataEngine()
+
+    def run(
+        self, request: GenerateRequest, log: Callable[[str], None]
+    ) -> dict[str, object]:
+        base_url = request.base_url or DEFAULT_BASE_URL
+        dataset_uri = (
+            self._engine.get_dataset_uri(request.api_key, base_url=base_url)
+            if request.api_key
+            else None
+        )
+        if not dataset_uri:
+            raise RuntimeError(
+                "WORDLIFT_KEY is required (or set wordlift.api_key in config)."
+            )
+        if not request.yarrrml_path.exists():
+            raise RuntimeError(f"YARRRML file not found: {request.yarrrml_path}")
+        yarrrml = request.yarrrml_path.read_text()
+
+        urls = resolve_input_urls(request.input_value)
+        urls = filter_urls(urls, request.regex, request.max_pages)
+
+        batch = BatchGenerator(
+            output_dir=request.output_dir,
+            output_format=request.output_format,
+            concurrency=request.concurrency,
+            headed=request.headed,
+            timeout_ms=request.timeout_ms,
+            wait_until=request.wait_until,
+            max_xhtml_chars=request.max_xhtml_chars,
+            max_text_node_chars=request.max_text_node_chars,
+            dataset_uri=dataset_uri,
+            verbose=request.verbose,
+        )
+        summary = batch.generate(urls, yarrrml, log)
+        summary["input"] = request.input_value
+        return summary
+
+
+def resolve_api_key_from_context(
+    ctx_config: object | None, env_key: str = "WORDLIFT_KEY"
+) -> str | None:
+    if ctx_config is not None:
+        try:
+            value = ctx_config.get("wordlift.api_key")
+            if value:
+                return value
+        except Exception:
+            return None
+    import os
+
+    return os.environ.get(env_key)

wordlift_sdk/structured_data/rendering.py

@@ -0,0 +1,43 @@
+"""Rendering and cleanup pipeline."""
+
+from __future__ import annotations
+
+from typing import Callable
+
+from wordlift_sdk.render import CleanupOptions, RenderOptions, clean_xhtml, render_html
+
+
+class RenderPipeline:
+    """Renders a page and cleans XHTML for structured data generation."""
+
+    def __init__(
+        self,
+        headed: bool,
+        timeout_ms: int,
+        wait_until: str,
+        max_xhtml_chars: int,
+        max_text_node_chars: int,
+    ) -> None:
+        self._headed = headed
+        self._timeout_ms = timeout_ms
+        self._wait_until = wait_until
+        self._max_xhtml_chars = max_xhtml_chars
+        self._max_text_node_chars = max_text_node_chars
+
+    def render(self, url: str, log: Callable[[str], None]) -> tuple[object, str]:
+        log("Rendering page with Playwright...")
+        render_options = RenderOptions(
+            url=url,
+            headless=not self._headed,
+            timeout_ms=self._timeout_ms,
+            wait_until=self._wait_until,
+        )
+        rendered = render_html(render_options)
+
+        log("Cleaning XHTML for prompt usage...")
+        cleanup_options = CleanupOptions(
+            max_xhtml_chars=self._max_xhtml_chars,
+            max_text_node_chars=self._max_text_node_chars,
+        )
+        cleaned_xhtml = clean_xhtml(rendered.xhtml, cleanup_options)
+        return rendered, cleaned_xhtml

wordlift_sdk/structured_data/schema_guide.py

@@ -0,0 +1,17 @@
+"""Schema.org shape and property guidance."""
+
+from __future__ import annotations
+
+from .engine import normalize_type, shape_specs_for_type
+
+
+class SchemaGuide:
+    """Builds schema.org property guides and shape specs."""
+
+    def normalize_type(self, value: str) -> str:
+        return normalize_type(value)
+
+    def shape_specs_for_type(self, type_name: str | None) -> list[str]:
+        if not type_name:
+            return []
+        return shape_specs_for_type(type_name)

wordlift_sdk/structured_data/structured_data_engine.py

@@ -0,0 +1,58 @@
+"""OOP facade for structured data utilities."""
+
+from __future__ import annotations
+
+from wordlift_sdk.structured_data.constants import DEFAULT_BASE_URL
+
+from .agent_generator import AgentGenerator
+from .dataset_resolver import DatasetResolver
+from .schema_guide import SchemaGuide
+from .yarrrml_pipeline import YarrrmlPipeline
+
+
+class StructuredDataEngine:
+    """OOP facade for structured data utilities."""
+
+    def __init__(
+        self,
+        dataset: DatasetResolver | None = None,
+        schema: SchemaGuide | None = None,
+        yarrrml: YarrrmlPipeline | None = None,
+        agent: AgentGenerator | None = None,
+    ) -> None:
+        self.dataset = dataset or DatasetResolver()
+        self.schema = schema or SchemaGuide()
+        self.yarrrml = yarrrml or YarrrmlPipeline()
+        self.agent = agent or AgentGenerator()
+
+    def get_dataset_uri(self, api_key: str, base_url: str = DEFAULT_BASE_URL) -> str:
+        return self.dataset.get_dataset_uri(api_key, base_url)
+
+    async def get_dataset_uri_async(
+        self, api_key: str, base_url: str = DEFAULT_BASE_URL
+    ) -> str:
+        return await self.dataset.get_dataset_uri_async(api_key, base_url)
+
+    def generate_from_agent(self, *args, **kwargs):
+        return self.agent.generate_from_agent(*args, **kwargs)
+
+    def normalize_yarrrml_mappings(self, *args, **kwargs):
+        return self.yarrrml.normalize_mappings(*args, **kwargs)
+
+    def materialize_yarrrml_jsonld(self, *args, **kwargs):
+        return self.yarrrml.materialize_jsonld(*args, **kwargs)
+
+    def postprocess_jsonld(self, *args, **kwargs):
+        return self.yarrrml.postprocess_jsonld(*args, **kwargs)
+
+    def make_reusable_yarrrml(self, *args, **kwargs):
+        return self.yarrrml.make_reusable_yarrrml(*args, **kwargs)
+
+    def ensure_no_blank_nodes(self, *args, **kwargs):
+        return self.yarrrml.ensure_no_blank_nodes(*args, **kwargs)
+
+    def build_output_basename(self, *args, **kwargs):
+        return self.yarrrml.build_output_basename(*args, **kwargs)
+
+    def shape_specs_for_type(self, *args, **kwargs):
+        return self.schema.shape_specs_for_type(*args, **kwargs)

wordlift_sdk/structured_data/validation.py

@@ -0,0 +1,31 @@
+"""SHACL validation helpers."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from wordlift_sdk.structured_data.schema_guide import SchemaGuide
+from wordlift_sdk.validation.shacl import validate_file
+
+
+class ValidationService:
+    """Validates JSON-LD outputs with SHACL shapes."""
+
+    def __init__(self, schema: SchemaGuide | None = None) -> None:
+        self._schema = schema or SchemaGuide()
+
+    def validate(self, jsonld_path: Path, target_type: str, workdir: Path) -> str:
+        shape_specs = self._schema.shape_specs_for_type(target_type)
+        result = validate_file(str(jsonld_path), shape_specs=shape_specs)
+        (workdir / "jsonld.validation.json").write_text(
+            json.dumps(
+                {
+                    "conforms": result.conforms,
+                    "warning_count": result.warning_count,
+                    "report_text": result.report_text,
+                },
+                indent=2,
+            )
+        )
+        return result.report_text

wordlift_sdk/structured_data/yarrrml_pipeline.py

@@ -0,0 +1,34 @@
+"""YARRRML to JSON-LD pipeline helpers."""
+
+from __future__ import annotations
+
+from .engine import (
+    build_output_basename,
+    ensure_no_blank_nodes,
+    make_reusable_yarrrml,
+    materialize_yarrrml_jsonld,
+    normalize_yarrrml_mappings,
+    postprocess_jsonld,
+)
+
+
+class YarrrmlPipeline:
+    """YARRRML -> JSON-LD pipeline helpers."""
+
+    def normalize_mappings(self, *args, **kwargs):
+        return normalize_yarrrml_mappings(*args, **kwargs)
+
+    def materialize_jsonld(self, *args, **kwargs):
+        return materialize_yarrrml_jsonld(*args, **kwargs)
+
+    def postprocess_jsonld(self, *args, **kwargs):
+        return postprocess_jsonld(*args, **kwargs)
+
+    def ensure_no_blank_nodes(self, *args, **kwargs):
+        return ensure_no_blank_nodes(*args, **kwargs)
+
+    def make_reusable_yarrrml(self, *args, **kwargs):
+        return make_reusable_yarrrml(*args, **kwargs)
+
+    def build_output_basename(self, *args, **kwargs):
+        return build_output_basename(*args, **kwargs)

wordlift_sdk/url_source/__init__.py

@@ -1,6 +1,11 @@
+__all__ = [
+    "Url",
+    "UrlSource",
+    "GoogleSheetsUrlSource",
+    "ListUrlSource",
+    "SitemapUrlSource",
+]
 from .google_sheets_url_source import GoogleSheetsUrlSource
 from .list_url_source import ListUrlSource
 from .sitemap_url_source import SitemapUrlSource
 from .url_source import UrlSource, Url
-
-__all__ = ["Url", "UrlSource", "GoogleSheetsUrlSource", "ListUrlSource", "SitemapUrlSource"]

wordlift_sdk/validation/__init__.py

@@ -0,0 +1,7 @@
+"""Validation utilities."""
+
+from __future__ import annotations
+
+from .shacl import ValidationResult, list_shape_names, validate_file
+
+__all__ = ["ValidationResult", "list_shape_names", "validate_file"]