wordlift-sdk 2.7.1__py3-none-any.whl

wordlift_sdk/wordlift/entity_gaps/create_entity_gaps_factory.py

@@ -0,0 +1,51 @@
+from typing import Callable, Awaitable, Optional
+
+from pandas import Series, DataFrame
+from pycountry import countries
+from tqdm.asyncio import tqdm
+from twisted.mail.scripts.mailmail import Configuration
+from wordlift_client import AccountInfo, AnalysesResponse
+
+from .entity_gaps_callback import entity_gaps_callback_factory
+from ...deprecated import create_entities_with_top_query_dataframe
+from ...utils import create_delayed
+
+
+async def create_entity_gaps_factory(
+    key: str, configuration: Configuration, account: AccountInfo
+):
+    async def callback(url_list: list[str]) -> DataFrame:
+        # Get the entity data with the top query.
+        entities_with_top_query_df = await create_entities_with_top_query_dataframe(
+            key=key, url_list=url_list
+        )
+
+        country = countries.get(alpha_2=account.country_code.upper())
+        delayed = create_delayed(
+            await append_entity_gaps_response_to_row_factory(
+                entity_gaps_callback=await entity_gaps_callback_factory(
+                    configuration=configuration, query_location_name=country.name
+                )
+            ),
+            2,
+        )
+        series = await tqdm.gather(
+            *[delayed(row) for index, row in entities_with_top_query_df.iterrows()],
+            total=len(entities_with_top_query_df),
+        )
+
+        return DataFrame(series)
+
+    return callback
+
+
+async def append_entity_gaps_response_to_row_factory(
+    entity_gaps_callback: Callable[[Series], Awaitable[Optional[AnalysesResponse]]],
+) -> Callable[[Series], Awaitable[Series]]:
+    async def append_entity_gaps_response_to_row(row: Series) -> Series:
+        response = await entity_gaps_callback(row)
+        if response:
+            row["entity_gaps"] = response.items
+        return row
+
+    return append_entity_gaps_response_to_row

wordlift_sdk/wordlift/entity_gaps/entity_gaps_callback.py

@@ -0,0 +1,32 @@
+from typing import Callable, Awaitable, Optional
+
+import wordlift_client
+from pandas import Series
+from tenacity import retry, stop_after_attempt, wait_fixed
+from twisted.mail.scripts.mailmail import Configuration
+from wordlift_client import AnalysesResponse, EntityGapsApi, EntityGapRequest
+
+
+async def entity_gaps_callback_factory(configuration: Configuration, query_location_name: str) -> Callable[
+    [Series], Awaitable[Optional[AnalysesResponse]]]:
+    @retry(
+        stop=stop_after_attempt(10),
+        wait=wait_fixed(2)
+    )
+    async def entity_gaps_callback(row: Series) -> Optional[AnalysesResponse]:
+        url = row['url']
+        query = row['top_query_name']
+        if query is None:
+            return None
+
+        async with wordlift_client.ApiClient(configuration) as api_client:
+            api = EntityGapsApi(api_client)
+            return await api.create_entity_gap(
+                EntityGapRequest(
+                    url=url,
+                    query=query,
+                    query_location_name=query_location_name
+                )
+            )
+
+    return entity_gaps_callback

wordlift_sdk/wordlift/sitemap_import/protocol/__init__.py

@@ -0,0 +1,14 @@
+from .default import DefaultImportUrlProtocol, DefaultParseHtmlProtocol
+from .import_url_protocol_interface import ImportUrlProtocolInterface, ImportUrlInput
+from .parse_html_protocol_interface import ParseHtmlProtocolInterface, ParseHtmlInput
+from .protocol_context import ProtocolContext
+
+__all__ = [
+    'ImportUrlProtocolInterface',
+    'ImportUrlInput',
+    'ParseHtmlProtocolInterface',
+    'ParseHtmlInput',
+    'ProtocolContext',
+    'DefaultImportUrlProtocol',
+    'DefaultParseHtmlProtocol',
+]

wordlift_sdk/wordlift/sitemap_import/protocol/default/__init__.py

@@ -0,0 +1,4 @@
+from .default_import_url_protocol import DefaultImportUrlProtocol
+from .default_parse_html_protocol import DefaultParseHtmlProtocol
+
+__all__ = ['DefaultParseHtmlProtocol', 'DefaultImportUrlProtocol']

wordlift_sdk/wordlift/sitemap_import/protocol/default/default_import_url_protocol.py

@@ -0,0 +1,39 @@
+import logging
+
+import wordlift_client
+from tenacity import retry, stop_after_attempt, wait_fixed
+from wordlift_client import SitemapImportsApi, SitemapImportRequest, EmbeddingRequest
+
+from ..import_url_protocol_interface import ImportUrlProtocolInterface, ImportUrlInput
+
+logger = logging.getLogger(__name__)
+
+
+class DefaultImportUrlProtocol(ImportUrlProtocolInterface):
+
+    @retry(
+        stop=stop_after_attempt(5),
+        wait=wait_fixed(2)
+    )
+    async def import_url(self, import_url_input: ImportUrlInput) -> None:
+        configuration = self.context.configuration
+        types = self.context.types
+        url_list = import_url_input.url_list
+
+        async with wordlift_client.ApiClient(configuration) as api_client:
+            imports_api = SitemapImportsApi(api_client)
+            request = SitemapImportRequest(
+                embedding=EmbeddingRequest(
+                    properties=["http://schema.org/headline", "http://schema.org/abstract",
+                                "http://schema.org/text"]
+                ),
+                output_types=list(types),
+                urls=list(url_list),
+                overwrite=True,
+                id_generator="headline-with-url-hash"
+            )
+
+            try:
+                await imports_api.create_sitemap_import(sitemap_import_request=request)
+            except Exception as e:
+                logger.error("Error importing URLs: %s", e)

wordlift_sdk/wordlift/sitemap_import/protocol/default/default_parse_html_protocol.py

@@ -0,0 +1,41 @@
+"""
+This module provides a default implementation of the ParseHtmlProtocolInterface.
+
+The DefaultParseHtmlProtocol class implements a basic HTML parsing protocol
+that can be used as a fallback or starting point for more complex implementations.
+"""
+
+from wordlift_client import EntityPatchRequest
+from ..parse_html_protocol_interface import ParseHtmlProtocolInterface, ParseHtmlInput
+
+
+class DefaultParseHtmlProtocol(ParseHtmlProtocolInterface):
+    """
+    Default implementation of the ParseHtmlProtocolInterface.
+
+    This class provides a minimal implementation of the HTML parsing protocol
+    that returns an empty list of entity patch requests. It can be used as a
+    base class for more complex implementations or as a fallback when no
+    specific parsing logic is required.
+
+    Attributes:
+        context (ProtocolContext): The protocol context containing configuration
+            and entity types information.
+    """
+
+    async def parse_html(self, parse_html_input: ParseHtmlInput) -> list[EntityPatchRequest]:
+        """
+        Parse HTML content and extract entity information.
+
+        This default implementation returns an empty list. Override this method
+        to implement custom HTML parsing logic.
+
+        Args:
+            parse_html_input (ParseHtmlInput): An object containing the HTML content to parse,
+                along with entity ID, URL, and additional data.
+
+        Returns:
+            list[EntityPatchRequest]: A list of entity patch requests to update
+                the knowledge graph. This implementation returns an empty list.
+        """
+        return list()

wordlift_sdk/wordlift/sitemap_import/protocol/import_url_protocol_interface.py

@@ -0,0 +1,21 @@
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import Protocol
+
+from .protocol_context import ProtocolContext
+
+
+@dataclass
+class ImportUrlInput:
+    url_list: list[str]
+
+
+class ImportUrlProtocolInterface(Protocol):
+    context: ProtocolContext
+
+    def __init__(self, context: ProtocolContext):
+        self.context = context
+
+    @abstractmethod
+    async def import_url(self, import_url_input: ImportUrlInput) -> None:
+        ...

wordlift_sdk/wordlift/sitemap_import/protocol/parse_html_protocol_interface.py

@@ -0,0 +1,27 @@
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import Protocol
+
+from pandas import Series
+from wordlift_client import EntityPatchRequest
+
+from .protocol_context import ProtocolContext
+
+
+@dataclass
+class ParseHtmlInput:
+    entity_id: str
+    entity_url: str
+    html: str
+    row: Series
+
+
+class ParseHtmlProtocolInterface(Protocol):
+    context: ProtocolContext
+
+    def __init__(self, context: ProtocolContext):
+        self.context = context
+
+    @abstractmethod
+    async def parse_html(self, parse_html_input: ParseHtmlInput) -> list[EntityPatchRequest]:
+        ...

wordlift_sdk/wordlift/sitemap_import/protocol/protocol_context.py

@@ -0,0 +1,16 @@
+from dataclasses import dataclass
+
+from wordlift_client import AccountInfo
+from wordlift_client import Configuration
+
+from ....id_generator.id_generator_interface import IdGeneratorInterface
+from ....protocol.graph import GraphQueue
+
+
+@dataclass
+class ProtocolContext:
+    account: AccountInfo
+    configuration: Configuration
+    id_generator: IdGeneratorInterface
+    types: list[str]
+    graph_queue: GraphQueue

wordlift_sdk/workflow/__init__.py

@@ -0,0 +1,3 @@
+from .kg_import_workflow import KgImportWorkflow
+
+__all__ = ["KgImportWorkflow"]

wordlift_sdk/workflow/create_or_update_entities_factory.py

@@ -0,0 +1,16 @@
+from wordlift_client import Configuration
+from rdflib import Graph
+import wordlift_client
+
+
+async def create_or_update_entities_factory(configuration: Configuration):
+    async def callback(graph: Graph):
+        # Run all the queued graphs.
+        async with wordlift_client.ApiClient(configuration=configuration) as api_client:
+            api_instance = wordlift_client.EntitiesApi(api_client)
+            await api_instance.create_or_update_entities(
+                graph.serialize(format="turtle"),
+                _content_type="text/turtle",
+            )
+
+    return callback

wordlift_sdk/workflow/kg_import_workflow.py

@@ -0,0 +1,49 @@
+import logging
+from os import cpu_count
+from pathlib import Path
+
+from tqdm.asyncio import tqdm
+
+from .url_handler.url_handler import UrlHandler
+from ..graph.ttl_liquid import TtlLiquidGraphFactory
+from ..protocol import (
+    Context,
+)
+from ..url_source import UrlSource
+from ..utils import create_delayed
+
+logger = logging.getLogger(__name__)
+
+
+class KgImportWorkflow:
+    _concurrency: int
+    _context: Context
+    _url_handler: UrlHandler
+    _url_source: UrlSource
+
+    def __init__(
+        self,
+        context: Context,
+        url_source: UrlSource,
+        url_handler: UrlHandler,
+        concurrency: int = min(cpu_count(), 4),
+    ) -> None:
+        self._context = context
+        self._url_source = url_source
+        self._url_handler = url_handler
+        self._concurrency = concurrency
+
+    async def run(self):
+        await TtlLiquidGraphFactory(
+            context=self._context, path=Path("data/templates")
+        ).graphs()
+
+        url_list = [url async for url in self._url_source.urls()]
+
+        logger.info("Applying %d URL import request(s)" % len(url_list))
+
+        delayed = create_delayed(self._url_handler, self._concurrency)
+        await tqdm.gather(
+            *[delayed(url) for url in list(url_list)],
+            total=len(url_list),
+        )

wordlift_sdk/workflow/patch_entities_factory.py

@@ -0,0 +1,16 @@
+import wordlift_client
+from wordlift_client import Configuration
+
+from ..protocol.entity_patch import EntityPatch
+
+
+async def patch_entities_factory(configuration: Configuration):
+    async def callback(entity_patch: EntityPatch):
+        # Run all the queued graphs.
+        async with wordlift_client.ApiClient(configuration=configuration) as api_client:
+            api_instance = wordlift_client.EntitiesApi(api_client)
+            await api_instance.patch_entities(
+                id=entity_patch.iri, entity=entity_patch.requests
+            )
+
+    return callback

wordlift_sdk/workflow/url_handler/__init__.py

@@ -0,0 +1,3 @@
+from .web_page_import_url_handler import WebPageImportUrlHandler
+
+__all__ = ["WebPageImportUrlHandler"]

wordlift_sdk/workflow/url_handler/default_url_handler.py

@@ -0,0 +1,23 @@
+import logging
+
+from ...url_source import Url
+from ...workflow.url_handler.url_handler import UrlHandler
+
+logger = logging.getLogger(__name__)
+
+
+class DefaultUrlHandler(UrlHandler):
+    _url_handler_list: list[UrlHandler]
+
+    def __init__(self, url_handler_list: list[UrlHandler]):
+        super().__init__()
+        self._url_handler_list = url_handler_list
+
+    async def __call__(self, url: Url) -> None:
+        for url_handler in self._url_handler_list:
+            try:
+                await url_handler.__call__(url)
+            except Exception as e:
+                logger.error(
+                    f"Handler {type(url_handler).__name__} errored while handling url {url}: {e}"
+                )

wordlift_sdk/workflow/url_handler/search_console_url_handler.py

@@ -0,0 +1,79 @@
+import asyncio
+import logging
+from datetime import datetime, timedelta
+
+import gql.transport.exceptions
+import aiohttp
+import pydantic_core
+import wordlift_client
+from tenacity import (
+    retry,
+    retry_if_exception_type,
+    wait_fixed,
+    after_log,
+    stop_after_attempt,
+)
+from wordlift_client import AnalyticsImportRequest
+
+from .url_handler import UrlHandler
+from ...graphql.client import GraphQlClient
+from ...graphql.utils.query import EntityTopQuery
+from ...protocol import Context
+from ...url_source import Url
+
+logger = logging.getLogger(__name__)
+
+
+class SearchConsoleUrlHandler(UrlHandler):
+    _context: Context
+    _graphql_client: GraphQlClient
+
+    def __init__(self, context: Context, graphql_client: GraphQlClient) -> None:
+        self._context = context
+        self._graphql_client = graphql_client
+
+    @retry(
+        retry=retry_if_exception_type(
+            asyncio.TimeoutError
+            | aiohttp.client_exceptions.ServerDisconnectedError
+            | aiohttp.client_exceptions.ClientConnectorError
+            | aiohttp.client_exceptions.ClientPayloadError
+            | aiohttp.client_exceptions.ClientConnectorDNSError
+            | pydantic_core._pydantic_core.ValidationError
+            | wordlift_client.exceptions.ServiceException
+            | wordlift_client.exceptions.BadRequestException
+            | aiohttp.client_exceptions.ClientOSError
+            | gql.transport.exceptions.TransportServerError
+        ),
+        wait=wait_fixed(2),  # Wait 2 seconds between retries
+        stop=stop_after_attempt(3),  # Max 3 retries
+        after=after_log(logger, logging.WARNING),
+        reraise=True,
+    )
+    async def __call__(self, url: Url) -> None:
+        if not self._context.account.google_search_console_site_url:
+            return
+
+        entities = await self._graphql_client.run(
+            graphql="entities_top_query", variables={"urls": [url.value]}
+        )
+
+        if not entities:
+            return
+
+        # Calculate the date 7 days ago from today
+        seven_days_ago = datetime.now() - timedelta(days=7)
+        entity_top_query = EntityTopQuery.from_graphql_response(entities[0])
+        if (
+            entity_top_query.top_query_date_created
+            and datetime.fromisoformat(entity_top_query.top_query_date_created)
+            > seven_days_ago
+        ):
+            return
+
+        async with wordlift_client.ApiClient(
+            self._context.client_configuration
+        ) as api_client:
+            api_instance = wordlift_client.AnalyticsImportsApi(api_client)
+            request = AnalyticsImportRequest(urls=[entity_top_query.url])
+            await api_instance.create_analytics_import(request)

wordlift_sdk/workflow/url_handler/url_handler.py

@@ -0,0 +1,8 @@
+from abc import ABC
+
+from ...url_source import Url
+
+
+class UrlHandler(ABC):
+    async def __call__(self, url: Url):
+        pass

wordlift_sdk/workflow/url_handler/web_page_import_url_handler.py

@@ -0,0 +1,104 @@
+import asyncio
+import logging
+
+import aiohttp
+import pydantic_core
+from tenacity import (
+    retry,
+    retry_if_exception_type,
+    wait_fixed,
+    after_log,
+    stop_after_attempt,
+)
+from wordlift_client import (
+    ApiClient,
+    WebPagesImportsApi,
+    WebPageImportRequest,
+    EmbeddingRequest,
+)
+import wordlift_client
+import gql.transport.exceptions
+
+from .url_handler import UrlHandler
+from ...protocol import (
+    Context,
+    WebPageImportProtocolInterface,
+    load_override_class,
+    DefaultWebPageImportProtocol,
+)
+from ...url_source import Url
+
+logger = logging.getLogger(__name__)
+
+
+class WebPageImportUrlHandler(UrlHandler):
+    _context: Context
+    _embedding_request: EmbeddingRequest
+    _web_page_import_callback: WebPageImportProtocolInterface
+    _web_page_types: list[str]
+    _write_strategy: str
+
+    def __init__(
+        self,
+        context: Context,
+        embedding_properties: list[str],
+        web_page_types: list[str],
+        web_page_import_callback: WebPageImportProtocolInterface | None = None,
+        write_strategy: str = "createOrUpdateModel",
+    ):
+        self._context = context
+        self._embedding_request = EmbeddingRequest(
+            properties=embedding_properties,
+        )
+        self._web_page_types = web_page_types
+        if web_page_import_callback is None:
+            self._web_page_import_callback = load_override_class(
+                name="web_page_import_protocol",
+                class_name="WebPageImportProtocol",
+                # Default class to use in case of missing override.
+                default_class=DefaultWebPageImportProtocol,
+                context=context,
+            )
+        else:
+            self._web_page_import_callback = web_page_import_callback
+
+        self._write_strategy = write_strategy
+
+    @retry(
+        retry=retry_if_exception_type(
+            asyncio.TimeoutError
+            | aiohttp.client_exceptions.ServerDisconnectedError
+            | aiohttp.client_exceptions.ClientConnectorError
+            | aiohttp.client_exceptions.ClientPayloadError
+            | aiohttp.client_exceptions.ClientConnectorDNSError
+            | pydantic_core._pydantic_core.ValidationError
+            | wordlift_client.exceptions.ServiceException
+            | gql.transport.exceptions.TransportServerError
+            | wordlift_client.exceptions.BadRequestException
+            | aiohttp.client_exceptions.ClientOSError
+        ),
+        wait=wait_fixed(2),  # Wait 2 seconds between retries
+        after=after_log(logger, logging.WARNING),
+        stop=stop_after_attempt(5),
+    )
+    async def __call__(self, url: Url) -> None:
+        async with ApiClient(self._context.client_configuration) as client:
+            api_instance = WebPagesImportsApi(client)
+
+            request = WebPageImportRequest(
+                url=url.value,
+                id=None if url.iri is None else url.iri,
+                embedding=self._embedding_request,
+                output_types=self._web_page_types,
+                id_generator="headline-with-url-hash",
+                write_strategy=self._write_strategy,
+            )
+
+            try:
+                response = await api_instance.create_web_page_imports(
+                    web_page_import_request=request, _request_timeout=120.0
+                )
+                await self._web_page_import_callback.callback(response)
+            except Exception as e:
+                logger.error("Error importing Web Page %s" % url.value, exc_info=e)
+                raise e