wordlift-sdk 2.7.1__py3-none-any.whl

wordlift_sdk/namespace/__init__.py

@@ -0,0 +1,3 @@
+from .SDO import SDO
+
+__all__ = ["SDO"]

wordlift_sdk/notebook/__init__.py

@@ -0,0 +1,3 @@
+from .install_if_missing import install_if_missing
+
+__all__ = ['install_if_missing']

wordlift_sdk/notebook/install_if_missing.py

@@ -0,0 +1,12 @@
+import logging
+import subprocess
+import sys
+
+
+def install_if_missing(package_spec: str, import_name=None):
+    import_name = import_name or package_spec.split()[0]
+    try:
+        __import__(import_name)
+    except ImportError:
+        print(f"{import_name} not found. Installing...")
+        subprocess.check_call([sys.executable, "-m", "pip", "install", package_spec])

wordlift_sdk/protocol/__init__.py

@@ -0,0 +1,5 @@
+from .context import Context
+from .load_override_class import load_override_class
+from .web_page_import_protocol import WebPageImportProtocolInterface, DefaultWebPageImportProtocol
+
+__all__ = ['Context', 'load_override_class', 'WebPageImportProtocolInterface', 'DefaultWebPageImportProtocol']

wordlift_sdk/protocol/context.py

@@ -0,0 +1,21 @@
+from dataclasses import dataclass
+
+from wordlift_client import AccountInfo, Configuration
+
+from .entity_patch import EntityPatchQueue
+from .graph import GraphQueue
+from ..configuration import ConfigurationProvider
+from ..id_generator import IdGenerator
+
+
+@dataclass
+class Context:
+    account: AccountInfo
+    client_configuration: Configuration
+    id_generator: IdGenerator
+
+    configuration_provider: ConfigurationProvider
+
+    # Queues where clients can append data to be written to the graph.
+    graph_queue: GraphQueue
+    entity_patch_queue: EntityPatchQueue

wordlift_sdk/protocol/entity_patch/__init__.py

@@ -0,0 +1,4 @@
+from .entity_patch import EntityPatch
+from .entity_patch_queue import EntityPatchQueue
+
+__all__ = ["EntityPatch", "EntityPatchQueue"]

wordlift_sdk/protocol/entity_patch/entity_patch.py

@@ -0,0 +1,8 @@
+from dataclasses import dataclass
+from wordlift_client import EntityPatchRequest
+
+
+@dataclass
+class EntityPatch:
+    iri: str
+    requests: list[EntityPatchRequest]

wordlift_sdk/protocol/entity_patch/entity_patch_queue.py

@@ -0,0 +1,49 @@
+import logging
+import asyncio
+import aiohttp
+import pydantic_core
+import wordlift_client
+from wordlift_client import Configuration
+
+from .entity_patch import EntityPatch
+from tenacity import retry, retry_if_exception_type, wait_fixed, after_log
+
+logger = logging.getLogger(__name__)
+
+
+class EntityPatchQueue:
+    client_configuration: Configuration
+
+    def __init__(self, client_configuration: Configuration):
+        self.client_configuration = client_configuration
+
+    @retry(
+        # stop=stop_after_attempt(5),  # Retry up to 5 times
+        retry=retry_if_exception_type(
+            asyncio.TimeoutError
+            | aiohttp.client_exceptions.ServerDisconnectedError
+            | aiohttp.client_exceptions.ClientConnectorError
+            | aiohttp.client_exceptions.ClientPayloadError
+            | aiohttp.client_exceptions.ClientConnectorDNSError
+            | pydantic_core._pydantic_core.ValidationError
+            | wordlift_client.exceptions.ServiceException
+            | wordlift_client.exceptions.BadRequestException
+            | aiohttp.client_exceptions.ClientOSError
+        ),
+        wait=wait_fixed(2),  # Wait 2 seconds between retries
+        after=after_log(logger, logging.WARNING),
+        reraise=True,
+    )
+    async def put(self, entity_patch: EntityPatch) -> None:
+        async with wordlift_client.ApiClient(
+            configuration=self.client_configuration
+        ) as api_client:
+            api_instance = wordlift_client.EntitiesApi(api_client)
+
+            try:
+                await api_instance.patch_entities(
+                    id=entity_patch.iri, entity_patch_request=entity_patch.requests
+                )
+            except Exception as e:
+                logger.error("Error patching entities", exc_info=e)
+                raise e

wordlift_sdk/protocol/graph/__init__.py

@@ -0,0 +1,3 @@
+from .graph_queue import GraphQueue
+
+__all__ = ["GraphQueue"]

wordlift_sdk/protocol/graph/graph_queue.py

@@ -0,0 +1,64 @@
+import hashlib
+import logging
+import aiohttp
+import asyncio
+
+import pydantic_core
+import wordlift_client
+from rdflib import Graph
+from rdflib.compare import to_isomorphic
+from wordlift_client import Configuration
+from tenacity import retry, retry_if_exception_type, wait_fixed, after_log
+
+logger = logging.getLogger(__name__)
+
+
+class GraphQueue:
+    client_configuration: Configuration
+    hashes: set[str]
+
+    def __init__(self, client_configuration: Configuration):
+        self.client_configuration = client_configuration
+        self.hashes = set()
+
+    @retry(
+        # stop=stop_after_attempt(5),  # Retry up to 5 times
+        retry=retry_if_exception_type(
+            asyncio.TimeoutError
+            | aiohttp.client_exceptions.ServerDisconnectedError
+            | aiohttp.client_exceptions.ClientConnectorError
+            | aiohttp.client_exceptions.ClientPayloadError
+            | aiohttp.client_exceptions.ClientConnectorDNSError
+            | pydantic_core._pydantic_core.ValidationError
+            | wordlift_client.exceptions.ServiceException
+            | wordlift_client.exceptions.BadRequestException
+            | aiohttp.client_exceptions.ClientOSError
+        ),
+        wait=wait_fixed(2),  # Wait 2 seconds between retries
+        after=after_log(logger, logging.WARNING),
+        reraise=True,
+    )
+    async def put(self, graph: Graph) -> None:
+        hash = GraphQueue.hash_graph(graph)
+        if hash not in self.hashes:
+            self.hashes.add(hash)
+
+            async with wordlift_client.ApiClient(
+                configuration=self.client_configuration
+            ) as api_client:
+                api_instance = wordlift_client.EntitiesApi(api_client)
+
+                try:
+                    await api_instance.create_or_update_entities(
+                        graph.serialize(format="turtle"),
+                        _content_type="text/turtle",
+                    )
+                except Exception as e:
+                    logger.error(f"Failed to create entities: {e}", exc_info=e)
+                    raise e
+
+    @staticmethod
+    def hash_graph(graph: Graph) -> str:
+        iso = to_isomorphic(graph)
+        canon = iso.serialize(format="nt")
+        return hashlib.sha256(canon.encode("utf-8")).hexdigest()

wordlift_sdk/protocol/load_override_class.py

@@ -0,0 +1,30 @@
+import os
+import sys
+from importlib.util import spec_from_file_location, module_from_spec
+from pathlib import Path
+from typing import Type, TypeVar
+
+T = TypeVar("T")
+
+
+def load_override_class(
+    name: str, class_name: str, default_class: Type[T], **kwargs
+) -> T:
+    override_dir = os.environ.get("WORDLIFT_OVERRIDE_DIR", "app/overrides")
+    override_path = Path(f"{override_dir}/{name}.py")
+
+    # Ensure the override directory is importable
+    abs_dir = str(Path(override_dir).resolve())
+    if abs_dir not in sys.path:
+        sys.path.insert(0, abs_dir)
+
+    if override_path.exists():
+        spec = spec_from_file_location(name, override_path)
+        mod = module_from_spec(spec)
+        sys.modules[name] = mod
+        spec.loader.exec_module(mod)
+
+        cls = getattr(mod, class_name)
+        return cls(**kwargs)
+
+    return default_class(**kwargs)

wordlift_sdk/protocol/web_page_import_protocol.py

@@ -0,0 +1,23 @@
+from abc import abstractmethod
+
+from .context import Context
+
+from typing import Protocol
+from wordlift_client import WebPageImportResponse
+
+
+class WebPageImportProtocolInterface(Protocol):
+    context: Context
+
+    def __init__(self, context: Context):
+        self.context = context
+
+    @abstractmethod
+    async def callback(self, web_page_import_response: WebPageImportResponse) -> None:
+        pass
+
+
+class DefaultWebPageImportProtocol(WebPageImportProtocolInterface):
+
+    async def callback(self, web_page_import_response: WebPageImportResponse) -> None:
+        pass

wordlift_sdk/url_source/__init__.py

@@ -0,0 +1,6 @@
+from .google_sheets_url_source import GoogleSheetsUrlSource
+from .list_url_source import ListUrlSource
+from .sitemap_url_source import SitemapUrlSource
+from .url_source import UrlSource, Url
+
+__all__ = ["Url", "UrlSource", "GoogleSheetsUrlSource", "ListUrlSource", "SitemapUrlSource"]

wordlift_sdk/url_source/google_sheets_url_source.py

@@ -0,0 +1,53 @@
+from typing import AsyncGenerator
+
+import pandas as pd
+from google.auth.credentials import Credentials
+from gspread import Client
+
+from .url_source import UrlSource, Url
+from ..utils.create_dataframe_from_google_sheets import create_dataframe_from_google_sheets
+
+
+class GoogleSheetsUrlSource(UrlSource):
+    """
+    A URL provider that extracts URLs from a Google Sheet.
+
+    This class implements the UrlProvider interface to provide URLs from a Google Sheet.
+    It uses the create_dataframe_from_google_sheets function to fetch the sheet data
+    and extracts the URLs from the 'url' column.
+    """
+
+    def __init__(self, creds_or_client: Credentials | Client, url: str, sheet: str):
+        """
+        Initialize the GoogleSheetsUrlProvider with Google Sheets credentials and URL.
+
+        Args:
+            creds_or_client (Credentials | Client): Google Auth Credentials or gspread Client
+            url (str): The URL of the Google Sheet
+            sheet (str): The name of the worksheet
+        """
+        self.creds_or_client = creds_or_client
+        self.url = url
+        self.sheet = sheet
+
+    async def urls(self) -> AsyncGenerator[Url, None]:
+        """
+        Asynchronously yield URLs from the Google Sheet.
+
+        This method fetches the Google Sheet data using the create_dataframe_from_google_sheets function,
+        and yields each URL from the 'url' column as a Url object.
+
+        Returns:
+            AsyncGenerator[Url, None]: An asynchronous generator that yields Url objects.
+        """
+        # Get the dataframe from the Google Sheet
+        df = create_dataframe_from_google_sheets(self.creds_or_client, self.url, self.sheet)
+
+        # Check if 'url' column exists
+        if 'url' not in df.columns:
+            raise ValueError("The Google Sheet must contain a 'url' column")
+
+        # Yield each URL from the 'url' column
+        for url in df['url']:
+            if pd.notna(url) and url.strip():  # Skip empty or NaN values
+                yield Url(value=url.strip())

wordlift_sdk/url_source/list_url_source.py

@@ -0,0 +1,28 @@
+from typing import AsyncGenerator
+
+from .url_source import UrlSource, Url
+
+
+class ListUrlSource(UrlSource):
+    """A URL provider that yields URLs from a predefined list.
+
+    This provider takes a list of URL strings and provides them one by one
+    through the async generator method `urls()`.
+    """
+
+    def __init__(self, urls: list[str]):
+        """Initialize the ListUrlProvider with a list of URLs.
+
+        Args:
+            urls: A list of URL strings to be provided.
+        """
+        self._url_list = urls
+
+    async def urls(self) -> AsyncGenerator[Url, None]:
+        """Asynchronously yield Url objects from the predefined list.
+
+        Yields:
+            Url: A Url object for each URL string in the list.
+        """
+        for url in self._url_list:
+            yield Url(value=url)

wordlift_sdk/url_source/new_or_changed_url_source.py

@@ -0,0 +1,57 @@
+from dataclasses import asdict
+from typing import AsyncGenerator
+
+import pandas as pd
+
+from . import UrlSource, Url
+from ..graphql.client import GraphQlClient
+
+
+class NewOrChangedUrlSource(UrlSource):
+    graphql_client: GraphQlClient
+    url_provider: UrlSource
+    overwrite: bool
+
+    def __init__(
+        self, url_provider: UrlSource, graphql_client: GraphQlClient, overwrite: bool
+    ):
+        self.graphql_client = graphql_client
+        self.url_provider = url_provider
+        self.overwrite = overwrite
+
+    async def urls(self) -> AsyncGenerator[Url, None]:
+        # Get the list of URLs from the underlying provider.
+        url_df = pd.DataFrame([asdict(url) async for url in self.url_provider.urls()])
+        # Get the list of URLs from GraphQL.
+        list_records = await self.graphql_client.run(
+            "entities_url_iri_with_source_equal_to_web_page_import",
+            {"urls": url_df["value"].tolist() if "value" in url_df.columns else []},
+        )
+        graphql_df = pd.DataFrame.from_records(
+            data=[record for record in list_records],
+            columns=("url", "iri", "date_imported"),
+        )
+        graphql_df["date_imported"] = pd.to_datetime(
+            graphql_df["date_imported"], utc=True, errors="coerce"
+        )
+        merged_df = pd.merge(
+            url_df,
+            graphql_df,
+            left_on="value",
+            right_on="url",
+            how="left",
+            suffixes=("", "_graphql"),
+        )
+        filtered_df = merged_df[
+            self.overwrite
+            | merged_df["date_imported"].isna()
+            | (merged_df["date_imported"] < merged_df["date_modified"])
+        ]
+        for _, row in filtered_df.iterrows():
+            yield Url(
+                value=row["value"],
+                iri=None if pd.isna(row["iri_graphql"]) else row["iri_graphql"],
+                date_modified=None
+                if pd.isna(row["date_modified"])
+                else row["date_modified"],
+            )

wordlift_sdk/url_source/sitemap_url_source.py

@@ -0,0 +1,36 @@
+import re
+from typing import AsyncGenerator, Optional
+
+import advertools as adv
+import pandas as pd
+
+from .url_source import UrlSource, Url
+
+
+class SitemapUrlSource(UrlSource):
+    sitemap_url: str
+    pattern: re.Pattern | None
+
+    def __init__(self, sitemap_url: str, pattern: Optional[re.Pattern] = None):
+        self.pattern = pattern
+        self.sitemap_url = sitemap_url
+
+    async def urls(self) -> AsyncGenerator[Url, None]:
+        sitemap_df = adv.sitemaps.sitemap_to_df(sitemap_url=self.sitemap_url)
+        # Ensure 'lastmod' column exists
+        if "lastmod" not in sitemap_df.columns:
+            sitemap_df["lastmod"] = None
+        sitemap_df["lastmod_as_datetime"] = pd.to_datetime(
+            sitemap_df["lastmod"], errors="coerce"
+        )
+
+        for _, row in sitemap_df.iterrows():
+            url = row["loc"]
+            last_mod_as_datetime = row["lastmod_as_datetime"]
+            if self.pattern is None or self.pattern.search(url):
+                yield Url(
+                    value=url,
+                    date_modified=None
+                    if pd.isna(last_mod_as_datetime)
+                    else last_mod_as_datetime.to_pydatetime(),
+                )

wordlift_sdk/url_source/url_source.py

@@ -0,0 +1,18 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from datetime import datetime
+from typing import AsyncGenerator
+
+
+@dataclass
+class Url:
+    value: str
+    iri: str | None = None
+    date_modified: datetime | None = None
+
+
+class UrlSource(ABC):
+    @abstractmethod
+    async def urls(self) -> AsyncGenerator[Url, None]:
+        """Asynchronously yields Url objects."""
+        pass

wordlift_sdk/url_source/url_source_input.py

@@ -0,0 +1,6 @@
+from dataclasses import dataclass
+from typing import Optional, List, Union
+
+from google.auth.credentials import Credentials
+from gspread import Client
+

wordlift_sdk/utils/__init__.py

@@ -0,0 +1,17 @@
+from .create_dataframe_from_google_sheets import create_dataframe_from_google_sheets
+from .create_dataframe_of_entities_by_types import create_dataframe_of_entities_by_types
+from .create_dataframe_of_entities_with_embedding_vectors import create_dataframe_of_entities_with_embedding_vectors
+from .create_dataframe_of_url_iri import create_dataframe_of_url_iri
+from .create_entity_patch_request import create_entity_patch_request
+from .delayed import create_delayed
+from .get_me import get_me
+
+__all__ = [
+    "create_dataframe_from_google_sheets",
+    "create_dataframe_of_entities_by_types",
+    "create_dataframe_of_entities_with_embedding_vectors",
+    "create_dataframe_of_url_iri",
+    "create_entity_patch_request",
+    "create_delayed",
+    "get_me"
+]

wordlift_sdk/utils/create_dataframe_from_google_sheets.py

@@ -0,0 +1,26 @@
+import gspread
+import pandas as pd
+from google.auth.credentials import Credentials
+from gspread import Client
+
+
+def create_dataframe_from_google_sheets(creds_or_client: Credentials | Client, url: str, sheet: str) -> pd.DataFrame:
+    if isinstance(creds_or_client, Credentials):
+        return create_dataframe_from_google_sheets_using_credentials(creds_or_client, url, sheet)
+    elif isinstance(creds_or_client, Client):
+        return create_dataframe_from_google_sheets_using_client(creds_or_client, url, sheet)
+    else:
+        raise TypeError("Expected creds_or_client to be of type Credentials or Client")
+
+
+def create_dataframe_from_google_sheets_using_credentials(creds: Credentials, url: str, sheet: str) -> pd.DataFrame:
+    gc = gspread.authorize(creds)
+
+    return create_dataframe_from_google_sheets_using_client(gc, url, sheet)
+
+
+def create_dataframe_from_google_sheets_using_client(gc: Client, url: str, sheet: str) -> pd.DataFrame:
+    sheet = gc.open_by_url(url).worksheet(sheet)
+    data = sheet.get_all_records()
+
+    return pd.DataFrame([{k.strip(): v for k, v in row.items()} for row in data])

wordlift_sdk/utils/create_dataframe_of_entities_by_types.py

@@ -0,0 +1,26 @@
+import pandas as pd
+
+from wordlift_sdk import graphql
+
+
+async def create_dataframe_of_entities_by_types(key: str, types: set[str]) -> pd.DataFrame:
+    return await graphql.query(
+        key=key,
+        query_string="""
+            query getEntities($types: [String]!) {
+              entities(
+                query: { typeConstraint: { in: $types } }
+              ) {
+                iri
+                keywords: string(name: "schema:keywords")
+                url: string(name: "schema:url")
+              }
+            }
+        """,
+        root_element="entities",
+        columns=['iri', 'keywords', 'url'],
+        variable_values={
+            # `set` cannot be serialized in Python, so we convert to `list`
+            "types": list(types)
+        }
+    )

wordlift_sdk/utils/create_dataframe_of_entities_with_embedding_vectors.py

@@ -0,0 +1,28 @@
+import pandas as pd
+from tenacity import retry, stop_after_attempt, wait_fixed
+
+from wordlift_sdk import graphql
+
+
+@retry(
+    stop=stop_after_attempt(5),
+    wait=wait_fixed(2)
+)
+async def create_dataframe_of_entities_with_embedding_vectors(key: str) -> pd.DataFrame:
+    return await graphql.query(
+        key=key,
+        query_string="""
+        query {
+          entities(
+            query: {
+              embeddingValueConstraint: { exists: { exists: true, excludeEmpty: true } }
+            }
+          ) {
+            iri
+            url: string(name: "schema:url")
+          }
+        }
+    """,
+        root_element='entities',
+        columns=['iri', 'url'],
+    )

wordlift_sdk/utils/create_dataframe_of_url_iri.py

@@ -0,0 +1,14 @@
+from typing import List
+
+import pandas as pd
+
+from ..graphql.client.factory import GraphQlClientFactory
+from wordlift_sdk.kg import EntityStoreFactory
+
+
+async def create_dataframe_of_url_iri(key: str, url_list: List[str]) -> pd.DataFrame:
+    graphql_client_factory = GraphQlClientFactory(key)
+    graphql_client = graphql_client_factory.create_gql_client()
+    entity_store_factory = EntityStoreFactory(graphql_client)
+    entity_store = entity_store_factory.create()
+    return await entity_store.url_iri_as_dataframe(url_list)

wordlift_sdk/utils/create_entity_patch_request.py

@@ -0,0 +1,14 @@
+from rdflib import Graph, URIRef, Node
+import wordlift_client
+from wordlift_client.models.entity_patch_request import EntityPatchRequest
+
+def create_entity_patch_request(resource: URIRef, prop: URIRef, value: Node) -> EntityPatchRequest:
+    g = Graph()
+    g.bind('schema', 'http://schema.org/')
+    g.add((resource, prop, value))
+
+    return wordlift_client.EntityPatchRequest(
+        op='add',
+        path='/' + str(prop),
+        value=g.serialize(format='json-ld', auto_compact=True)
+    )

wordlift_sdk/utils/delayed.py

@@ -0,0 +1,12 @@
+import asyncio
+from multiprocessing import cpu_count
+
+
+def create_delayed(callback, concurrency=cpu_count() + 1):
+    sem = asyncio.Semaphore(concurrency)
+
+    async def callback_with_semaphore(*args, **kwargs):
+        async with sem:
+            return await callback(*args, **kwargs)
+
+    return callback_with_semaphore

wordlift_sdk/utils/get_me.py

@@ -0,0 +1,8 @@
+import wordlift_client
+from wordlift_client import Configuration, AccountInfo, AccountApi
+
+
+async def get_me(configuration: Configuration) -> AccountInfo:
+    async with wordlift_client.ApiClient(configuration) as api_client:
+        api = AccountApi(api_client)
+        return await api.get_me()

wordlift_sdk/utils/import_url.py

@@ -0,0 +1,35 @@
+import logging
+from typing import Callable, Awaitable
+
+from tenacity import retry, stop_after_attempt, wait_fixed
+from wordlift_client import Configuration, SitemapImportsApi, SitemapImportRequest, EmbeddingRequest
+
+logger = logging.getLogger(__name__)
+
+
+async def import_url_factory(configuration: Configuration, types: set[str]) -> Callable[[set[str]], Awaitable[None]]:
+    @retry(
+        stop=stop_after_attempt(5),
+        wait=wait_fixed(2)
+    )
+    async def import_url(url_list: set[str]) -> None:
+        import wordlift_client
+
+        async with wordlift_client.ApiClient(configuration) as api_client:
+            imports_api = SitemapImportsApi(api_client)
+            request = SitemapImportRequest(
+                embedding=EmbeddingRequest(
+                    properties=["http://schema.org/headline", "http://schema.org/abstract", "http://schema.org/text"]
+                ),
+                output_types=list(types),
+                urls=list(url_list),
+                overwrite=True,
+                id_generator="headline-with-url-hash"
+            )
+
+            try:
+                await imports_api.create_sitemap_import(sitemap_import_request=request)
+            except Exception as e:
+                logger.error("Error importing URLs: %s", e)
+
+    return import_url

wordlift_sdk/wordlift/entity_gaps/__init__.py

@@ -0,0 +1,3 @@
+from .create_entity_gaps_factory import create_entity_gaps_factory
+
+__all__ = ["create_entity_gaps_factory"]