wordlift-sdk 2.7.1__py3-none-any.whl

wordlift_sdk/__init__.py

@@ -0,0 +1,3 @@
+from .main import run_kg_import_workflow
+
+__all__ = ['run_kg_import_workflow']

wordlift_sdk/client/__init__.py

@@ -0,0 +1,3 @@
+from .client_configuration_factory import ClientConfigurationFactory
+
+__all__ = ['ClientConfigurationFactory']

wordlift_sdk/client/client_configuration_factory.py

@@ -0,0 +1,26 @@
+import wordlift_client
+
+
+class ClientConfigurationFactory:
+    _api_url: str
+    _key: str
+
+    def __init__(self, key: str, api_url: str = "https://api.wordlift.io"):
+        self._api_url = api_url
+        self._key = key
+
+    def create(self):
+        configuration = wordlift_client.Configuration(
+            host=self._api_url,
+        )
+
+        # The client must configure the authentication and authorization parameters
+        # in accordance with the API server security policy.
+        # Examples for each auth method are provided below, use the example that
+        # satisfies your auth use case.
+
+        # Configure API key authorization: ApiKey
+        configuration.api_key['ApiKey'] = self._key
+        configuration.api_key_prefix['ApiKey'] = 'Key'
+
+        return configuration

wordlift_sdk/configuration/__init__.py

@@ -0,0 +1,4 @@
+from .configuration_provider import ConfigurationProvider
+from .get_config_value import get_config_value
+
+__all__ = ['ConfigurationProvider', 'get_config_value']

wordlift_sdk/configuration/configuration_provider.py

@@ -0,0 +1,44 @@
+import importlib.util
+import os
+
+
+class ConfigurationProvider:
+    _config: dict
+
+    @staticmethod
+    def create(filepath: str = "config/default.py") -> "ConfigurationProvider":
+        return ConfigurationProvider(filepath=filepath)
+
+    def __init__(self, filepath: str):
+        if not os.path.exists(filepath):
+            self._config = {}
+        spec = importlib.util.spec_from_file_location("local_config", filepath)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        self._config = {k: getattr(module, k) for k in dir(module) if not k.startswith("_")}
+
+    def get_value(self, key: str, default=None):
+        # 1. Check globals
+        if key in globals():
+            return globals()[key]
+
+        # 2. Check config.py
+        if key in self._config:
+            return self._config[key]
+
+        # 3. Check environment variables
+        import os
+        if key in os.environ:
+            return os.environ[key]
+
+        # 4. Check Google Colab userdata
+        try:
+            from google.colab import userdata
+            secret = userdata.get(key)
+            if secret is not None:
+                return secret
+        except ImportError:
+            pass  # Not running in Google Colab
+
+        # 5. Return default if provided
+        return default

wordlift_sdk/configuration/get_config_value.py

@@ -0,0 +1,39 @@
+import os
+import importlib.util
+
+
+def load_config_py(filepath="config.py"):
+    if not os.path.exists(filepath):
+        return {}
+    spec = importlib.util.spec_from_file_location("local_config", filepath)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return {k: getattr(module, k) for k in dir(module) if not k.startswith("_")}
+
+
+def get_config_value(key, config_py_path=None, default=None):
+    # 1. Check globals
+    if key in globals():
+        return globals()[key]
+
+    # 2. Check config.py
+    config = load_config_py(config_py_path)
+    if key in config:
+        return config[key]
+
+    # 3. Check environment variables
+    import os
+    if key in os.environ:
+        return os.environ[key]
+
+    # 4. Check Google Colab userdata
+    try:
+        from google.colab import userdata
+        secret = userdata.get(key)
+        if secret is not None:
+            return secret
+    except ImportError:
+        pass  # Not running in Google Colab
+
+    # 5. Return default if provided
+    return default

wordlift_sdk/container/__init__.py

@@ -0,0 +1,3 @@
+from .application_container import ApplicationContainer
+
+__all__ = ['ApplicationContainer']

wordlift_sdk/container/application_container.py

@@ -0,0 +1,234 @@
+import re
+from dataclasses import dataclass
+from os import cpu_count
+from typing import Optional, Union
+
+import gspread
+from google.auth.credentials import Credentials
+from gspread import Client
+from wordlift_client import Configuration, AccountInfo
+
+from ..client.client_configuration_factory import ClientConfigurationFactory
+from ..configuration import ConfigurationProvider
+from ..graphql.client import GraphQlClientFactory, GraphQlClient, GqlClientProvider
+from ..id_generator import IdGenerator
+from ..protocol import Context
+from ..protocol.entity_patch import EntityPatchQueue
+from ..protocol.graph import GraphQueue
+from ..url_source import (
+    SitemapUrlSource,
+    GoogleSheetsUrlSource,
+    ListUrlSource,
+    UrlSource,
+)
+from ..url_source.new_or_changed_url_source import NewOrChangedUrlSource
+from ..utils import get_me
+from ..workflow.kg_import_workflow import KgImportWorkflow
+from ..workflow.url_handler import WebPageImportUrlHandler
+from ..workflow.url_handler.default_url_handler import DefaultUrlHandler
+from ..workflow.url_handler.search_console_url_handler import SearchConsoleUrlHandler
+from ..workflow.url_handler.url_handler import UrlHandler
+
+
+@dataclass
+class UrlSourceInput:
+    """
+    Input structure for the UrlProviderFactory.
+
+    This class holds all possible parameters needed to create any of the supported URL providers.
+    The factory will use these parameters to determine which provider to create based on availability.
+    """
+
+    sitemap_url: Optional[str] = None
+    sitemap_url_pattern: Optional[str] = None
+    sheets_url: Optional[str] = None
+    sheets_name: Optional[str] = None
+    sheets_creds_or_client: Optional[Union[Credentials, Client]] = None
+    urls: Optional[list[str]] = None
+
+
+class ApplicationContainer:
+    _api_url: str
+    _client_configuration: Configuration
+    _configuration_provider: ConfigurationProvider
+    _key: str
+
+    _context: Context | None = None
+    _graphql_client: GraphQlClient | None = None
+
+    def __init__(self, configuration_provider: ConfigurationProvider | None = None):
+        self._configuration_provider = (
+            configuration_provider or ConfigurationProvider.create()
+        )
+        self._api_url = self._configuration_provider.get_value(
+            "API_URL", "https://api.wordlift.io"
+        )
+        self._key = self._configuration_provider.get_value("WORDLIFT_KEY")
+        self._client_configuration = ClientConfigurationFactory(
+            key=self._key,
+            api_url=self._api_url,
+        ).create()
+
+    async def get_account(self) -> AccountInfo:
+        return await get_me(configuration=self._client_configuration)
+
+    async def get_context(self) -> Context:
+        if not self._context:
+            account = await self.get_account()
+            self._context = Context(
+                account=account,
+                client_configuration=self._client_configuration,
+                configuration_provider=self._configuration_provider,
+                id_generator=IdGenerator(account=account),
+                graph_queue=GraphQueue(client_configuration=self._client_configuration),
+                entity_patch_queue=EntityPatchQueue(
+                    client_configuration=self._client_configuration
+                ),
+            )
+
+        return self._context
+
+    async def create_web_page_import_url_handler(self) -> WebPageImportUrlHandler:
+        write_strategy = self._configuration_provider.get_value(
+            "WEB_PAGE_IMPORT_WRITE_STRATEGY", "createOrUpdateModel"
+        )
+        return WebPageImportUrlHandler(
+            context=await self.get_context(),
+            embedding_properties=self._configuration_provider.get_value(
+                "EMBEDDING_PROPERTIES",
+                [
+                    "http://schema.org/headline",
+                    "http://schema.org/abstract",
+                    "http://schema.org/text",
+                ],
+            ),
+            web_page_types=self._configuration_provider.get_value(
+                "WEB_PAGE_TYPES", ["http://schema.org/Article"]
+            ),
+            write_strategy=write_strategy,
+        )
+
+    async def create_search_console_url_handler(self):
+        return SearchConsoleUrlHandler(
+            context=await self.get_context(),
+            graphql_client=await self.get_graphql_client(),
+        )
+
+    async def create_multi_url_handler(self):
+        handlers: list[UrlHandler] = [
+            await self.create_web_page_import_url_handler(),
+        ]
+        if (
+            self._configuration_provider.get_value("GOOGLE_SEARCH_CONSOLE", True)
+            is True
+        ):
+            handlers.append(await self.create_search_console_url_handler())
+
+        return DefaultUrlHandler(url_handler_list=handlers)
+
+    async def create_kg_import_workflow(self) -> KgImportWorkflow:
+        concurrency = self._configuration_provider.get_value(
+            "CONCURRENCY", min(cpu_count(), 4)
+        )
+        return KgImportWorkflow(
+            context=await self.get_context(),
+            url_source=await self.create_new_or_changed_source(),
+            url_handler=await self.create_multi_url_handler(),
+            concurrency=concurrency,
+        )
+
+    async def create_graphql_client_factory(self) -> GraphQlClientFactory:
+        return GraphQlClientFactory(key=self._key, api_url=self._api_url + "/graphql")
+
+    async def create_gql_client_provider(self) -> GqlClientProvider:
+        graphql_client_factory = await self.create_graphql_client_factory()
+        return graphql_client_factory.create_provider()
+
+    async def get_graphql_client(self) -> GraphQlClient:
+        if self._graphql_client is None:
+            graphql_client_factory = await self.create_graphql_client_factory()
+            self._graphql_client = graphql_client_factory.create()
+
+        return self._graphql_client
+
+    async def create_url_source(self) -> UrlSource:
+        # Try to read the configuration from the `config/default.py` file.
+        sitemap_url = self._configuration_provider.get_value("SITEMAP_URL")
+        sitemap_url_pattern = self._configuration_provider.get_value(
+            "SITEMAP_URL_PATTERN", None
+        )
+        sheets_url = self._configuration_provider.get_value("SHEETS_URL")
+        sheets_name = self._configuration_provider.get_value("SHEETS_NAME")
+        sheets_service_account = self._configuration_provider.get_value(
+            "SHEETS_SERVICE_ACCOUNT"
+        )
+        urls = self._configuration_provider.get_value("URLS")
+
+        if (
+            sitemap_url is None
+            and urls is None
+            and (
+                sheets_url is None
+                or sheets_name is None
+                or sheets_service_account is None
+            )
+        ):
+            raise ValueError(
+                "One of `sitemap_url` or `sheets_url`/`sheets_name`/`sheets_service_account` is required."
+            )
+
+        input_params = UrlSourceInput(
+            sitemap_url=sitemap_url,
+            sitemap_url_pattern=sitemap_url_pattern,
+            sheets_url=sheets_url,
+            sheets_name=sheets_name,
+            sheets_creds_or_client=(
+                gspread.service_account(filename=sheets_service_account)
+                if sheets_service_account
+                else None
+            ),
+            urls=urls,
+        )
+
+        # Try to create a SitemapUrlProvider if sitemap_url is provided
+        if input_params.sitemap_url:
+            return SitemapUrlSource(
+                input_params.sitemap_url,
+                re.compile(input_params.sitemap_url_pattern)
+                if input_params.sitemap_url_pattern
+                else None,
+            )
+
+        # Try to create a GoogleSheetsUrlProvider if all required sheets parameters are provided
+        if (
+            input_params.sheets_url
+            and input_params.sheets_name
+            and input_params.sheets_creds_or_client
+        ):
+            return GoogleSheetsUrlSource(
+                input_params.sheets_creds_or_client,
+                input_params.sheets_url,
+                input_params.sheets_name,
+            )
+
+        # Try to create a ListUrlProvider if urls is provided
+        if input_params.urls:
+            return ListUrlSource(input_params.urls)
+
+        # If we get here, none of the required parameters were provided
+        raise ValueError(
+            "No valid parameters provided to create a URL provider. "
+            "Please provide either sitemap_url, all sheets parameters "
+            "(sheets_url, sheets_name, sheets_creds_or_client), or urls."
+        )
+
+    async def create_new_or_changed_source(self) -> UrlSource:
+        overwrite = self._configuration_provider.get_value("OVERWRITE", False)
+        return NewOrChangedUrlSource(
+            url_provider=await self.create_url_source(),
+            graphql_client=await self.get_graphql_client(),
+            overwrite=overwrite,
+        )
+
+    async def create_url_source_with_overwrite(self) -> UrlSource:
+        return await self.create_new_or_changed_source()

wordlift_sdk/deprecated/__init__.py

@@ -0,0 +1,5 @@
+from .create_entities_with_top_query_dataframe import (
+    create_entities_with_top_query_dataframe,
+)
+
+__all__ = ["create_entities_with_top_query_dataframe"]

wordlift_sdk/deprecated/create_entities_with_top_query_dataframe.py

@@ -0,0 +1,30 @@
+import logging
+
+import pandas as pd
+from pandas import DataFrame
+from tqdm.asyncio import tqdm
+
+from ..graphql.utils.query import entity_with_top_query_factory
+from ..utils import create_delayed
+
+logger = logging.getLogger(__name__)
+
+
+async def create_entities_with_top_query_dataframe(
+    key: str, url_list: list[str]
+) -> DataFrame:
+    # Get the entities data with the top query.
+    logger.info("Loading entities with top query...")
+    entity_with_top_query = await entity_with_top_query_factory(key)
+    delayed = create_delayed(entity_with_top_query, 4)
+    entities_with_top_query = await tqdm.gather(
+        *[delayed(url) for url in url_list], total=len(url_list)
+    )
+
+    # Get a list of dataframes.
+    dataframes = [
+        obj.to_dataframe() for obj in entities_with_top_query if obj is not None
+    ]
+
+    # Concat them together, with a new index.
+    return pd.concat(dataframes, ignore_index=True)

wordlift_sdk/entity/__init__.py

@@ -0,0 +1,4 @@
+from .enrich import enrich, ParseHtmlCallback
+from .patch import patch
+
+__all__ = ['enrich', 'ParseHtmlCallback', 'patch']

wordlift_sdk/entity/enrich.py

@@ -0,0 +1,54 @@
+import logging
+from typing import Callable, Awaitable, Coroutine
+
+from aiohttp import ClientSession
+from pandas import Series
+from tenacity import retry, stop_after_attempt, wait_fixed
+from wordlift_client import EntityPatchRequest, Configuration, WebPagesApi
+import wordlift_client
+
+from .patch import patch
+from ..wordlift.sitemap_import.protocol.parse_html_protocol_interface import ParseHtmlInput
+
+logger = logging.getLogger(__name__)
+
+ParseHtmlCallback = Callable[[ParseHtmlInput], Awaitable[list[EntityPatchRequest]]]
+
+headers = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/123.0.0.0 Safari/537.36"
+    ),
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.5",
+}
+
+
+def enrich(configuration: Configuration, callback: ParseHtmlCallback) -> Callable[
+    [Series], Coroutine[None, None, None]]:
+    @retry(
+        stop=stop_after_attempt(5),  # Retry up to 5 times
+        wait=wait_fixed(2)  # Wait 2 seconds between retries
+    )
+    async def process(row: Series) -> None:
+        entity_url = row['url']
+        entity_id = row['iri']
+
+        async with wordlift_client.ApiClient(configuration) as api_client:
+            try:
+                api_instance = WebPagesApi(api_client=api_client)
+                web_page = await api_instance.get_web_page(entity_url)
+                html = web_page.html
+                parse_html_input = ParseHtmlInput(
+                    entity_id=entity_id,
+                    entity_url=entity_url,
+                    html=html,
+                    row=row
+                )
+                payloads = await callback(parse_html_input)
+                await patch(configuration, entity_id, payloads)
+            except Exception as e:
+                logger.error("Error %s occurred while processing entity %s with url %s" % (e, entity_id, entity_url))
+
+    return process

wordlift_sdk/entity/patch.py

@@ -0,0 +1,14 @@
+from typing import List, Optional
+
+import wordlift_client
+from wordlift_client import Configuration, EntityPatchRequest
+
+
+async def patch(configuration: Configuration, entity_id: str, payloads: List[EntityPatchRequest]) -> Optional[str]:
+    # If the payloads are empty, exit.
+    if not payloads:
+        return None
+
+    async with wordlift_client.ApiClient(configuration) as api_client:
+        api_instance = wordlift_client.EntitiesApi(api_client)
+        return await api_instance.patch_entities(entity_id, payloads)

wordlift_sdk/google_search_console/__init__.py

@@ -0,0 +1,5 @@
+from .create_google_search_console_data_import import create_google_search_console_data_import, import_url_analytics_factory
+from .raise_error_if_account_analytics_not_configured import raise_error_if_account_analytics_not_configured
+
+__all__ = ['create_google_search_console_data_import', 'import_url_analytics_factory',
+           'raise_error_if_account_analytics_not_configured']

wordlift_sdk/google_search_console/create_google_search_console_data_import.py

@@ -0,0 +1,59 @@
+import logging
+from datetime import datetime, timedelta
+from typing import Callable, Awaitable
+
+import wordlift_client
+from pandas import Series
+from tenacity import retry, wait_fixed, stop_after_attempt
+from tqdm.asyncio import tqdm
+from twisted.mail.scripts.mailmail import Configuration
+from wordlift_client import AnalyticsImportRequest
+
+from ..deprecated import create_entities_with_top_query_dataframe
+from ..utils import create_delayed
+
+logger = logging.getLogger(__name__)
+
+
+async def create_google_search_console_data_import(
+    configuration: Configuration, key: str, url_list: list[str]
+) -> None:
+    # Get the entities data with the top query.
+    entities_with_top_query_df = await create_entities_with_top_query_dataframe(
+        key=key, url_list=url_list
+    )
+
+    # Calculate the date 7 days ago from today
+    seven_days_ago = datetime.now() - timedelta(days=7)
+
+    # Filter the DataFrame
+    entities_with_stale_data_df = entities_with_top_query_df[
+        entities_with_top_query_df["top_query_date_created"].isna()
+        | (entities_with_top_query_df["top_query_date_created"] < seven_days_ago)
+    ]
+
+    import_url_analytics = await import_url_analytics_factory(
+        configuration=configuration
+    )
+    if len(entities_with_stale_data_df) > 0:
+        logger.info("Updating missing or stale Google Search Console data...")
+        # We're polite and not making more than 2 concurrent reqs.
+        delayed = create_delayed(import_url_analytics, 2)
+        await tqdm.gather(
+            *[delayed(row) for index, row in entities_with_stale_data_df.iterrows()],
+            total=len(entities_with_stale_data_df),
+        )
+
+
+async def import_url_analytics_factory(
+    configuration: Configuration,
+) -> Callable[[Series], Awaitable[None]]:
+    @retry(stop=stop_after_attempt(5), wait=wait_fixed(2))
+    async def import_url_analytics(row: Series) -> None:
+        url = row["url"]
+        async with wordlift_client.ApiClient(configuration) as api_client:
+            api_instance = wordlift_client.AnalyticsImportsApi(api_client)
+            request = AnalyticsImportRequest(urls=[url])
+            await api_instance.create_analytics_import(request)
+
+    return import_url_analytics

wordlift_sdk/google_search_console/raise_error_if_account_analytics_not_configured.py

@@ -0,0 +1,20 @@
+from pycountry import countries
+from wordlift_client import AccountInfo
+
+
+async def raise_error_if_account_analytics_not_configured(account: AccountInfo) -> bool:
+    if account.google_search_console_site_url is None:
+        raise ValueError(
+            "%s is not connected to Google Search Console, open https://my.wordlift.io to connect it." % account.dataset_uri)
+
+    if account.country_code is None:
+        raise ValueError(
+            "%s country code not configured, open https://my.wordlift.io to configure it." % account.dataset_uri)
+
+    # Get the country name
+    country = countries.get(alpha_2=account.country_code.upper())
+    if country is None:
+        raise ValueError(
+            "Country code %s is invalid, open https://my.wordlift.io to reconfigure it." % account.country_code.upper())
+
+    return True

wordlift_sdk/graph/graph_bag.py

@@ -0,0 +1,7 @@
+from dataclasses import dataclass
+from rdflib import Graph
+
+
+@dataclass
+class GraphBag:
+    graph: Graph

wordlift_sdk/graph/ttl_liquid/__init__.py

@@ -0,0 +1,3 @@
+from .ttl_liquid_graph_factory import TtlLiquidGraphFactory
+
+__all__ = ['TtlLiquidGraphFactory']

wordlift_sdk/graph/ttl_liquid/ttl_liquid_graph_factory.py

@@ -0,0 +1,43 @@
+import logging
+from pathlib import Path
+
+from liquid import Environment, CachingFileSystemLoader
+from rdflib import Graph
+
+from ...protocol import Context
+
+logger = logging.getLogger(__name__)
+
+
+class TtlLiquidGraphFactory:
+    path: Path
+    context: Context
+
+    def __init__(self, context: Context, path: Path):
+        self.context = context
+        self.path = path
+
+    async def graphs(self) -> None:
+        templates = list(self.path.rglob("*.ttl.liquid"))
+        env = Environment(
+            loader=CachingFileSystemLoader(self.path),
+        )
+
+        for template in templates:
+            template = env.get_template(str(template.absolute()))
+            turtle = template.render(account=self.context.account.__dict__)
+
+            try:
+                # Create a new RDF graph
+                graph = Graph()
+
+                # Parse the Turtle data into the graph
+                graph.parse(data=turtle, format="turtle")
+
+                logger.info(
+                    f"Successfully loaded {template} graph with {len(graph)} triples"
+                )
+
+                await self.context.graph_queue.put(graph)
+            except Exception as e:
+                logger.error(f"Error loading contact points graph: {e}")

wordlift_sdk/graphql/__init__.py

@@ -0,0 +1,3 @@
+from .query import query
+
+__all__ = ['query']

wordlift_sdk/graphql/client/__init__.py

@@ -0,0 +1,5 @@
+from .client import GraphQlClient
+from .factory import GraphQlClientFactory
+from .gql_client_provider import GqlClientProvider
+
+__all__ = ["GraphQlClient", "GraphQlClientFactory", "GqlClientProvider"]