wordlift-sdk 2.7.5__tar.gz

wordlift_sdk-2.7.5/PKG-INFO

@@ -0,0 +1,125 @@
+Metadata-Version: 2.4
+Name: wordlift-sdk
+Version: 2.7.5
+Summary: 
+Author: David Riccitelli
+Author-email: david@wordlift.io
+Requires-Python: >=3.10,<3.14
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: advertools (>0.16.6,<1.0.0)
+Requires-Dist: aiohttp (>=3.10.5,<4.0.0)
+Requires-Dist: google-auth (>=2.35.0,<3.0.0)
+Requires-Dist: gql[aiohttp] (>=3.5.2,<4.0.0)
+Requires-Dist: gspread (>=6.1.2,<7.0.0)
+Requires-Dist: pandas (>=2.1.4,<2.3.0)
+Requires-Dist: pycountry (>=24.6.1,<25.0.0)
+Requires-Dist: python-liquid (>=2.0.1,<3.0.0)
+Requires-Dist: rdflib (>=7.0.0,<8.0.0)
+Requires-Dist: tenacity (>=9.0.0,<10.0.0)
+Requires-Dist: tqdm (>=4.67.1,<5.0.0)
+Requires-Dist: wordlift-client (>=1.117.0,<2.0.0)
+Description-Content-Type: text/markdown
+
+ # WordLift Python SDK
+
+A Python toolkit for orchestrating WordLift imports: fetch URLs from sitemaps, Google Sheets, or explicit lists, filter out already imported pages, enqueue search console jobs, push RDF graphs, and call the WordLift APIs to import web pages.
+
+## Features
+- URL sources: XML sitemaps (with optional regex filtering), Google Sheets (`url` column), or Python lists.
+- Change detection: skips URLs that are already imported unless `OVERWRITE` is enabled; re-imports when `lastmod` is newer.
+- Web page imports: sends URLs to WordLift with embedding requests, output types, retry logic, and pluggable callbacks.
+- Search Console refresh: triggers analytics imports when top queries are stale.
+- Graph templates: renders `.ttl.liquid` templates under `data/templates` with account data and uploads the resulting RDF graphs.
+- Extensible: override protocols via `WORDLIFT_OVERRIDE_DIR` without changing the library code.
+
+## Installation
+
+```bash
+pip install wordlift-sdk
+# or
+poetry add wordlift-sdk
+```
+
+Requires Python 3.10–3.13.
+
+## Configuration
+
+Settings are read in order: `config/default.py` (or a custom path you pass to `ConfigurationProvider.create`), environment variables, then (when available) Google Colab `userdata`.
+
+Common options:
+- `WORDLIFT_KEY` (required): WordLift API key.
+- `API_URL`: WordLift API base URL, defaults to `https://api.wordlift.io`.
+- `SITEMAP_URL`: XML sitemap to crawl; `SITEMAP_URL_PATTERN` optional regex to filter URLs.
+- `SHEETS_URL`, `SHEETS_NAME`, `SHEETS_SERVICE_ACCOUNT`: use a Google Sheet as source; service account points to credentials file.
+- `URLS`: list of URLs (e.g., `["https://example.com/a", "https://example.com/b"]`).
+- `OVERWRITE`: re-import URLs even if already present (default `False`).
+- `WEB_PAGE_IMPORT_WRITE_STRATEGY`: WordLift write strategy (default `createOrUpdateModel`).
+- `EMBEDDING_PROPERTIES`: list of schema properties to embed.
+- `WEB_PAGE_TYPES`: output schema types, defaults to `["http://schema.org/Article"]`.
+- `GOOGLE_SEARCH_CONSOLE`: enable/disable Search Console handler (default `True`).
+- `CONCURRENCY`: max concurrent handlers, defaults to `min(cpu_count(), 4)`.
+- `WORDLIFT_OVERRIDE_DIR`: folder containing protocol overrides (default `app/overrides`).
+
+Example `config/default.py`:
+
+```python
+WORDLIFT_KEY = "your-api-key"
+SITEMAP_URL = "https://example.com/sitemap.xml"
+SITEMAP_URL_PATTERN = r"^https://example.com/article/.*$"
+GOOGLE_SEARCH_CONSOLE = True
+WEB_PAGE_TYPES = ["http://schema.org/Article"]
+EMBEDDING_PROPERTIES = [
+    "http://schema.org/headline",
+    "http://schema.org/abstract",
+    "http://schema.org/text",
+]
+```
+
+## Running the import workflow
+
+```python
+import asyncio
+from wordlift_sdk import run_kg_import_workflow
+
+if __name__ == "__main__":
+    asyncio.run(run_kg_import_workflow())
+```
+
+The workflow:
+1. Renders and uploads RDF graphs from `data/templates/*.ttl.liquid` using account info.
+2. Builds the configured URL source and filters out unchanged URLs (unless `OVERWRITE`).
+3. Sends each URL to WordLift for import with retries and optional Search Console refresh.
+
+You can build components yourself when you need more control:
+
+```python
+import asyncio
+from wordlift_sdk.container.application_container import ApplicationContainer
+
+async def main():
+    container = ApplicationContainer()
+    workflow = await container.create_kg_import_workflow()
+    await workflow.run()
+
+asyncio.run(main())
+```
+
+## Custom callbacks and overrides
+
+Override the web page import callback by placing `web_page_import_protocol.py` with a `WebPageImportProtocol` class under `WORDLIFT_OVERRIDE_DIR` (default `app/overrides`). The callback receives a `WebPageImportResponse` and can push to `graph_queue` or `entity_patch_queue`.
+
+## Templates
+
+Add `.ttl.liquid` files under `data/templates`. Templates render with `account` fields available (e.g., `{{ account.dataset_uri }}`) and are uploaded before URL handling begins.
+
+## Testing
+
+```bash
+poetry install --with dev
+poetry run pytest
+```
+

wordlift_sdk-2.7.5/README.md

@@ -0,0 +1,98 @@
+ # WordLift Python SDK
+
+A Python toolkit for orchestrating WordLift imports: fetch URLs from sitemaps, Google Sheets, or explicit lists, filter out already imported pages, enqueue search console jobs, push RDF graphs, and call the WordLift APIs to import web pages.
+
+## Features
+- URL sources: XML sitemaps (with optional regex filtering), Google Sheets (`url` column), or Python lists.
+- Change detection: skips URLs that are already imported unless `OVERWRITE` is enabled; re-imports when `lastmod` is newer.
+- Web page imports: sends URLs to WordLift with embedding requests, output types, retry logic, and pluggable callbacks.
+- Search Console refresh: triggers analytics imports when top queries are stale.
+- Graph templates: renders `.ttl.liquid` templates under `data/templates` with account data and uploads the resulting RDF graphs.
+- Extensible: override protocols via `WORDLIFT_OVERRIDE_DIR` without changing the library code.
+
+## Installation
+
+```bash
+pip install wordlift-sdk
+# or
+poetry add wordlift-sdk
+```
+
+Requires Python 3.10–3.13.
+
+## Configuration
+
+Settings are read in order: `config/default.py` (or a custom path you pass to `ConfigurationProvider.create`), environment variables, then (when available) Google Colab `userdata`.
+
+Common options:
+- `WORDLIFT_KEY` (required): WordLift API key.
+- `API_URL`: WordLift API base URL, defaults to `https://api.wordlift.io`.
+- `SITEMAP_URL`: XML sitemap to crawl; `SITEMAP_URL_PATTERN` optional regex to filter URLs.
+- `SHEETS_URL`, `SHEETS_NAME`, `SHEETS_SERVICE_ACCOUNT`: use a Google Sheet as source; service account points to credentials file.
+- `URLS`: list of URLs (e.g., `["https://example.com/a", "https://example.com/b"]`).
+- `OVERWRITE`: re-import URLs even if already present (default `False`).
+- `WEB_PAGE_IMPORT_WRITE_STRATEGY`: WordLift write strategy (default `createOrUpdateModel`).
+- `EMBEDDING_PROPERTIES`: list of schema properties to embed.
+- `WEB_PAGE_TYPES`: output schema types, defaults to `["http://schema.org/Article"]`.
+- `GOOGLE_SEARCH_CONSOLE`: enable/disable Search Console handler (default `True`).
+- `CONCURRENCY`: max concurrent handlers, defaults to `min(cpu_count(), 4)`.
+- `WORDLIFT_OVERRIDE_DIR`: folder containing protocol overrides (default `app/overrides`).
+
+Example `config/default.py`:
+
+```python
+WORDLIFT_KEY = "your-api-key"
+SITEMAP_URL = "https://example.com/sitemap.xml"
+SITEMAP_URL_PATTERN = r"^https://example.com/article/.*$"
+GOOGLE_SEARCH_CONSOLE = True
+WEB_PAGE_TYPES = ["http://schema.org/Article"]
+EMBEDDING_PROPERTIES = [
+    "http://schema.org/headline",
+    "http://schema.org/abstract",
+    "http://schema.org/text",
+]
+```
+
+## Running the import workflow
+
+```python
+import asyncio
+from wordlift_sdk import run_kg_import_workflow
+
+if __name__ == "__main__":
+    asyncio.run(run_kg_import_workflow())
+```
+
+The workflow:
+1. Renders and uploads RDF graphs from `data/templates/*.ttl.liquid` using account info.
+2. Builds the configured URL source and filters out unchanged URLs (unless `OVERWRITE`).
+3. Sends each URL to WordLift for import with retries and optional Search Console refresh.
+
+You can build components yourself when you need more control:
+
+```python
+import asyncio
+from wordlift_sdk.container.application_container import ApplicationContainer
+
+async def main():
+    container = ApplicationContainer()
+    workflow = await container.create_kg_import_workflow()
+    await workflow.run()
+
+asyncio.run(main())
+```
+
+## Custom callbacks and overrides
+
+Override the web page import callback by placing `web_page_import_protocol.py` with a `WebPageImportProtocol` class under `WORDLIFT_OVERRIDE_DIR` (default `app/overrides`). The callback receives a `WebPageImportResponse` and can push to `graph_queue` or `entity_patch_queue`.
+
+## Templates
+
+Add `.ttl.liquid` files under `data/templates`. Templates render with `account` fields available (e.g., `{{ account.dataset_uri }}`) and are uploaded before URL handling begins.
+
+## Testing
+
+```bash
+poetry install --with dev
+poetry run pytest
+```

wordlift_sdk-2.7.5/pyproject.toml

@@ -0,0 +1,40 @@
+[tool.poetry]
+name = "wordlift-sdk"
+version = "2.7.5"
+description = ""
+authors = ["David Riccitelli <david@wordlift.io>"]
+readme = "README.md"
+packages = [{ include = "wordlift_sdk" }]
+include = ["wordlift_sdk/graphql/data/**"]
+
+[tool.poetry.dependencies]
+python = ">=3.10, <3.14"
+gql = { extras = ["aiohttp"], version = "^3.5.2" }
+tenacity = "^9.0.0"
+aiohttp = "^3.10.5"
+# Google Colab requires pandas 2.1.4
+pandas = ">=2.1.4, <2.3.0"
+rdflib = "^7.0.0"
+wordlift-client = "^1.117.0"
+gspread = "^6.1.2"
+google-auth = "^2.35.0"
+tqdm = "^4.67.1"
+advertools = ">0.16.6,<1.0.0"
+pycountry = "^24.6.1"
+python-liquid = "^2.0.1"
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.3.3"
+pytest-asyncio = "^0.24.0"
+docker = "^7.1.0"
+ruff = "^0.11.13"
+pre-commit = "^4.2.0"
+pandas-stubs = "^2.3.3.251201"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+# see https://github.com/pytest-dev/pytest-asyncio/issues/924#issuecomment-2321921915
+[tool.pytest.ini_options]
+asyncio_default_fixture_loop_scope = "function"

wordlift_sdk-2.7.5/wordlift_sdk/__init__.py

@@ -0,0 +1,3 @@
+from .main import run_kg_import_workflow
+
+__all__ = ['run_kg_import_workflow']

wordlift_sdk-2.7.5/wordlift_sdk/client/__init__.py

@@ -0,0 +1,3 @@
+from .client_configuration_factory import ClientConfigurationFactory
+
+__all__ = ['ClientConfigurationFactory']

wordlift_sdk-2.7.5/wordlift_sdk/client/client_configuration_factory.py

@@ -0,0 +1,26 @@
+import wordlift_client
+
+
+class ClientConfigurationFactory:
+    _api_url: str
+    _key: str
+
+    def __init__(self, key: str, api_url: str = "https://api.wordlift.io"):
+        self._api_url = api_url
+        self._key = key
+
+    def create(self):
+        configuration = wordlift_client.Configuration(
+            host=self._api_url,
+        )
+
+        # The client must configure the authentication and authorization parameters
+        # in accordance with the API server security policy.
+        # Examples for each auth method are provided below, use the example that
+        # satisfies your auth use case.
+
+        # Configure API key authorization: ApiKey
+        configuration.api_key['ApiKey'] = self._key
+        configuration.api_key_prefix['ApiKey'] = 'Key'
+
+        return configuration

wordlift_sdk-2.7.5/wordlift_sdk/configuration/__init__.py

@@ -0,0 +1,4 @@
+from .configuration_provider import ConfigurationProvider
+from .get_config_value import get_config_value
+
+__all__ = ['ConfigurationProvider', 'get_config_value']

wordlift_sdk-2.7.5/wordlift_sdk/configuration/configuration_provider.py

@@ -0,0 +1,44 @@
+import importlib.util
+import os
+
+
+class ConfigurationProvider:
+    _config: dict
+
+    @staticmethod
+    def create(filepath: str = "config/default.py") -> "ConfigurationProvider":
+        return ConfigurationProvider(filepath=filepath)
+
+    def __init__(self, filepath: str):
+        if not os.path.exists(filepath):
+            self._config = {}
+        spec = importlib.util.spec_from_file_location("local_config", filepath)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        self._config = {k: getattr(module, k) for k in dir(module) if not k.startswith("_")}
+
+    def get_value(self, key: str, default=None):
+        # 1. Check globals
+        if key in globals():
+            return globals()[key]
+
+        # 2. Check config.py
+        if key in self._config:
+            return self._config[key]
+
+        # 3. Check environment variables
+        import os
+        if key in os.environ:
+            return os.environ[key]
+
+        # 4. Check Google Colab userdata
+        try:
+            from google.colab import userdata
+            secret = userdata.get(key)
+            if secret is not None:
+                return secret
+        except ImportError:
+            pass  # Not running in Google Colab
+
+        # 5. Return default if provided
+        return default

wordlift_sdk-2.7.5/wordlift_sdk/configuration/get_config_value.py

@@ -0,0 +1,39 @@
+import os
+import importlib.util
+
+
+def load_config_py(filepath="config.py"):
+    if not os.path.exists(filepath):
+        return {}
+    spec = importlib.util.spec_from_file_location("local_config", filepath)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return {k: getattr(module, k) for k in dir(module) if not k.startswith("_")}
+
+
+def get_config_value(key, config_py_path=None, default=None):
+    # 1. Check globals
+    if key in globals():
+        return globals()[key]
+
+    # 2. Check config.py
+    config = load_config_py(config_py_path)
+    if key in config:
+        return config[key]
+
+    # 3. Check environment variables
+    import os
+    if key in os.environ:
+        return os.environ[key]
+
+    # 4. Check Google Colab userdata
+    try:
+        from google.colab import userdata
+        secret = userdata.get(key)
+        if secret is not None:
+            return secret
+    except ImportError:
+        pass  # Not running in Google Colab
+
+    # 5. Return default if provided
+    return default

wordlift_sdk-2.7.5/wordlift_sdk/container/__init__.py

@@ -0,0 +1,3 @@
+from .application_container import ApplicationContainer
+
+__all__ = ['ApplicationContainer']

wordlift_sdk-2.7.5/wordlift_sdk/container/application_container.py

@@ -0,0 +1,234 @@
+import re
+from dataclasses import dataclass
+from os import cpu_count
+from typing import Optional, Union
+
+import gspread
+from google.auth.credentials import Credentials
+from gspread import Client
+from wordlift_client import Configuration, AccountInfo
+
+from ..client.client_configuration_factory import ClientConfigurationFactory
+from ..configuration import ConfigurationProvider
+from ..graphql.client import GraphQlClientFactory, GraphQlClient, GqlClientProvider
+from ..id_generator import IdGenerator
+from ..protocol import Context
+from ..protocol.entity_patch import EntityPatchQueue
+from ..protocol.graph import GraphQueue
+from ..url_source import (
+    SitemapUrlSource,
+    GoogleSheetsUrlSource,
+    ListUrlSource,
+    UrlSource,
+)
+from ..url_source.new_or_changed_url_source import NewOrChangedUrlSource
+from ..utils import get_me
+from ..workflow.kg_import_workflow import KgImportWorkflow
+from ..workflow.url_handler import WebPageImportUrlHandler
+from ..workflow.url_handler.default_url_handler import DefaultUrlHandler
+from ..workflow.url_handler.search_console_url_handler import SearchConsoleUrlHandler
+from ..workflow.url_handler.url_handler import UrlHandler
+
+
+@dataclass
+class UrlSourceInput:
+    """
+    Input structure for the UrlProviderFactory.
+
+    This class holds all possible parameters needed to create any of the supported URL providers.
+    The factory will use these parameters to determine which provider to create based on availability.
+    """
+
+    sitemap_url: Optional[str] = None
+    sitemap_url_pattern: Optional[str] = None
+    sheets_url: Optional[str] = None
+    sheets_name: Optional[str] = None
+    sheets_creds_or_client: Optional[Union[Credentials, Client]] = None
+    urls: Optional[list[str]] = None
+
+
+class ApplicationContainer:
+    _api_url: str
+    _client_configuration: Configuration
+    _configuration_provider: ConfigurationProvider
+    _key: str
+
+    _context: Context | None = None
+    _graphql_client: GraphQlClient | None = None
+
+    def __init__(self, configuration_provider: ConfigurationProvider | None = None):
+        self._configuration_provider = (
+            configuration_provider or ConfigurationProvider.create()
+        )
+        self._api_url = self._configuration_provider.get_value(
+            "API_URL", "https://api.wordlift.io"
+        )
+        self._key = self._configuration_provider.get_value("WORDLIFT_KEY")
+        self._client_configuration = ClientConfigurationFactory(
+            key=self._key,
+            api_url=self._api_url,
+        ).create()
+
+    async def get_account(self) -> AccountInfo:
+        return await get_me(configuration=self._client_configuration)
+
+    async def get_context(self) -> Context:
+        if not self._context:
+            account = await self.get_account()
+            self._context = Context(
+                account=account,
+                client_configuration=self._client_configuration,
+                configuration_provider=self._configuration_provider,
+                id_generator=IdGenerator(account=account),
+                graph_queue=GraphQueue(client_configuration=self._client_configuration),
+                entity_patch_queue=EntityPatchQueue(
+                    client_configuration=self._client_configuration
+                ),
+            )
+
+        return self._context
+
+    async def create_web_page_import_url_handler(self) -> WebPageImportUrlHandler:
+        write_strategy = self._configuration_provider.get_value(
+            "WEB_PAGE_IMPORT_WRITE_STRATEGY", "createOrUpdateModel"
+        )
+        return WebPageImportUrlHandler(
+            context=await self.get_context(),
+            embedding_properties=self._configuration_provider.get_value(
+                "EMBEDDING_PROPERTIES",
+                [
+                    "http://schema.org/headline",
+                    "http://schema.org/abstract",
+                    "http://schema.org/text",
+                ],
+            ),
+            web_page_types=self._configuration_provider.get_value(
+                "WEB_PAGE_TYPES", ["http://schema.org/Article"]
+            ),
+            write_strategy=write_strategy,
+        )
+
+    async def create_search_console_url_handler(self):
+        return SearchConsoleUrlHandler(
+            context=await self.get_context(),
+            graphql_client=await self.get_graphql_client(),
+        )
+
+    async def create_multi_url_handler(self):
+        handlers: list[UrlHandler] = [
+            await self.create_web_page_import_url_handler(),
+        ]
+        if (
+            self._configuration_provider.get_value("GOOGLE_SEARCH_CONSOLE", True)
+            is True
+        ):
+            handlers.append(await self.create_search_console_url_handler())
+
+        return DefaultUrlHandler(url_handler_list=handlers)
+
+    async def create_kg_import_workflow(self) -> KgImportWorkflow:
+        concurrency = self._configuration_provider.get_value(
+            "CONCURRENCY", min(cpu_count(), 4)
+        )
+        return KgImportWorkflow(
+            context=await self.get_context(),
+            url_source=await self.create_new_or_changed_source(),
+            url_handler=await self.create_multi_url_handler(),
+            concurrency=concurrency,
+        )
+
+    async def create_graphql_client_factory(self) -> GraphQlClientFactory:
+        return GraphQlClientFactory(key=self._key, api_url=self._api_url + "/graphql")
+
+    async def create_gql_client_provider(self) -> GqlClientProvider:
+        graphql_client_factory = await self.create_graphql_client_factory()
+        return graphql_client_factory.create_provider()
+
+    async def get_graphql_client(self) -> GraphQlClient:
+        if self._graphql_client is None:
+            graphql_client_factory = await self.create_graphql_client_factory()
+            self._graphql_client = graphql_client_factory.create()
+
+        return self._graphql_client
+
+    async def create_url_source(self) -> UrlSource:
+        # Try to read the configuration from the `config/default.py` file.
+        sitemap_url = self._configuration_provider.get_value("SITEMAP_URL")
+        sitemap_url_pattern = self._configuration_provider.get_value(
+            "SITEMAP_URL_PATTERN", None
+        )
+        sheets_url = self._configuration_provider.get_value("SHEETS_URL")
+        sheets_name = self._configuration_provider.get_value("SHEETS_NAME")
+        sheets_service_account = self._configuration_provider.get_value(
+            "SHEETS_SERVICE_ACCOUNT"
+        )
+        urls = self._configuration_provider.get_value("URLS")
+
+        if (
+            sitemap_url is None
+            and urls is None
+            and (
+                sheets_url is None
+                or sheets_name is None
+                or sheets_service_account is None
+            )
+        ):
+            raise ValueError(
+                "One of `sitemap_url` or `sheets_url`/`sheets_name`/`sheets_service_account` is required."
+            )
+
+        input_params = UrlSourceInput(
+            sitemap_url=sitemap_url,
+            sitemap_url_pattern=sitemap_url_pattern,
+            sheets_url=sheets_url,
+            sheets_name=sheets_name,
+            sheets_creds_or_client=(
+                gspread.service_account(filename=sheets_service_account)
+                if sheets_service_account
+                else None
+            ),
+            urls=urls,
+        )
+
+        # Try to create a SitemapUrlProvider if sitemap_url is provided
+        if input_params.sitemap_url:
+            return SitemapUrlSource(
+                input_params.sitemap_url,
+                re.compile(input_params.sitemap_url_pattern)
+                if input_params.sitemap_url_pattern
+                else None,
+            )
+
+        # Try to create a GoogleSheetsUrlProvider if all required sheets parameters are provided
+        if (
+            input_params.sheets_url
+            and input_params.sheets_name
+            and input_params.sheets_creds_or_client
+        ):
+            return GoogleSheetsUrlSource(
+                input_params.sheets_creds_or_client,
+                input_params.sheets_url,
+                input_params.sheets_name,
+            )
+
+        # Try to create a ListUrlProvider if urls is provided
+        if input_params.urls:
+            return ListUrlSource(input_params.urls)
+
+        # If we get here, none of the required parameters were provided
+        raise ValueError(
+            "No valid parameters provided to create a URL provider. "
+            "Please provide either sitemap_url, all sheets parameters "
+            "(sheets_url, sheets_name, sheets_creds_or_client), or urls."
+        )
+
+    async def create_new_or_changed_source(self) -> UrlSource:
+        overwrite = self._configuration_provider.get_value("OVERWRITE", False)
+        return NewOrChangedUrlSource(
+            url_provider=await self.create_url_source(),
+            graphql_client=await self.get_graphql_client(),
+            overwrite=overwrite,
+        )
+
+    async def create_url_source_with_overwrite(self) -> UrlSource:
+        return await self.create_new_or_changed_source()

wordlift_sdk-2.7.5/wordlift_sdk/deprecated/__init__.py

@@ -0,0 +1,5 @@
+from .create_entities_with_top_query_dataframe import (
+    create_entities_with_top_query_dataframe,
+)
+
+__all__ = ["create_entities_with_top_query_dataframe"]

wordlift_sdk-2.7.5/wordlift_sdk/deprecated/create_entities_with_top_query_dataframe.py

@@ -0,0 +1,30 @@
+import logging
+
+import pandas as pd
+from pandas import DataFrame
+from tqdm.asyncio import tqdm
+
+from ..graphql.utils.query import entity_with_top_query_factory
+from ..utils import create_delayed
+
+logger = logging.getLogger(__name__)
+
+
+async def create_entities_with_top_query_dataframe(
+    key: str, url_list: list[str]
+) -> DataFrame:
+    # Get the entities data with the top query.
+    logger.info("Loading entities with top query...")
+    entity_with_top_query = await entity_with_top_query_factory(key)
+    delayed = create_delayed(entity_with_top_query, 4)
+    entities_with_top_query = await tqdm.gather(
+        *[delayed(url) for url in url_list], total=len(url_list)
+    )
+
+    # Get a list of dataframes.
+    dataframes = [
+        obj.to_dataframe() for obj in entities_with_top_query if obj is not None
+    ]
+
+    # Concat them together, with a new index.
+    return pd.concat(dataframes, ignore_index=True)

wordlift_sdk-2.7.5/wordlift_sdk/entity/__init__.py

@@ -0,0 +1,4 @@
+from .enrich import enrich, ParseHtmlCallback
+from .patch import patch
+
+__all__ = ['enrich', 'ParseHtmlCallback', 'patch']