talisman-api 1.0.0__py3-none-any.whl

talisman_api/__init__.py

@@ -0,0 +1,11 @@
+__all__ = [
+    'AbstractTalismanAPI',
+    'APISchema', 'CompositeTalismanAPIClient',
+    'version',
+    'TalismanAPIClient'
+]
+
+from .abstract import AbstractTalismanAPI
+from .api_client import APISchema, CompositeTalismanAPIClient
+from .decorator import version
+from .wrapper import TalismanAPIClient

talisman_api/abstract.py

@@ -0,0 +1,91 @@
+import inspect
+import logging
+from abc import ABCMeta, abstractmethod
+from importlib import import_module, resources
+from typing import Iterable
+
+from gql import gql
+from graphql import DocumentNode, GraphQLError
+from typing_extensions import Self
+
+from .api_client import CompositeTalismanAPIClient
+from .api_client.composite import APISchema
+from .wrapper import TalismanAPIClient
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class AbstractTalismanAPI(metaclass=ABCMeta):
+    _VERSION: str
+
+    def __init__(self, client: CompositeTalismanAPIClient, documents: dict[APISchema, DocumentNode]):
+        if set(client) != set(documents):
+            raise ValueError
+        self._composite_client: CompositeTalismanAPIClient = client
+        self._documents: dict[APISchema, DocumentNode] = documents
+
+    def _client(self, schema: APISchema) -> TalismanAPIClient:
+        return TalismanAPIClient(self._composite_client[schema], self._documents[schema])
+
+    @classmethod
+    @abstractmethod
+    def _required_apis(cls) -> Iterable[APISchema]:
+        pass
+
+    @classmethod
+    async def create(cls, client: CompositeTalismanAPIClient) -> Self:
+        documents: dict[APISchema, DocumentNode] = {}
+        for api_schema in cls._required_apis():
+            if api_schema not in client:
+                raise ValueError(f'Client is not configured to use {api_schema} API')
+
+            package, _ = cls.__module__.rsplit('.', maxsplit=1)
+            resource_name = f'graphql/{cls._VERSION}/{api_schema.value}.graphql'
+            try:
+                with (resources.files(package) / resource_name).open('r', encoding='utf-8') as f:
+                    document: str = f.read()
+            except Exception as e:
+                raise ValueError(f'Package {package} contains no required {resource_name} file') from e
+            try:
+                document: DocumentNode = gql(document)
+            except GraphQLError as e:
+                raise ValueError(f'GraphQL syntax error if {resource_name} (package {package})') from e
+
+            api_client = client[api_schema]
+            async with api_client:
+                if not api_client.validate(document):
+                    raise ValueError(f'{resource_name} (package {package}) do not match runtime server {api_schema} API')
+            documents[api_schema] = document
+        return cls(client, documents)
+
+    @classmethod
+    async def get_compatible_api(
+            cls,
+            client: CompositeTalismanAPIClient,
+            *,
+            logger: logging.Logger = _LOGGER,
+            impl_package: str = '_impl'
+    ) -> Self:
+        package, _ = cls.__module__.rsplit('.', 1)
+        package_name = f'{package}.{impl_package}'
+        package = import_module(package_name)
+        if not hasattr(package, '__all__'):
+            raise ValueError(
+                f'{package_name}.__init__ should define __all__ attribute with ordered API implementation classes'
+            )
+        for impl_name in package.__all__:
+            impl_cls = getattr(package, impl_name, None)
+            if not inspect.isclass(impl_cls):
+                logger.error(f'{package_name}.__all__ should contains only API implementation classes')
+                continue
+            if not issubclass(impl_cls, cls):
+                logger.error(f'{package_name}.{impl_name} should inherit {cls}')
+                continue
+            try:
+                return await impl_cls.create(client)
+            except ValueError as e:
+                logger.info(
+                    f'{package_name}.{impl_name} (version: "{impl_cls._VERSION}") is not compatible with actual API: {e}',
+                    exc_info=e
+                )
+        raise NotImplementedError('API version is not supported')

talisman_api/api_client/__init__.py

@@ -0,0 +1,9 @@
+__all__ = [
+    'TalismanAPIClient',
+    'APISchema', 'CompositeTalismanAPIClient',
+    'GQLClientConfig'
+]
+
+from .client import TalismanAPIClient
+from .composite import APISchema, CompositeTalismanAPIClient
+from .gql_clients import GQLClientConfig

talisman_api/api_client/client.py

@@ -0,0 +1,112 @@
+import asyncio
+import logging
+from contextlib import AbstractAsyncContextManager
+from typing import AsyncIterator, Callable
+
+from graphql import DocumentNode
+from requests import Timeout
+
+from .gql_clients import AsyncAbstractGQLClient, GQLClientConfig
+
+logger = logging.getLogger(__name__)
+
+
+class TalismanAPIClient(AbstractAsyncContextManager):
+
+    def __init__(self, config: GQLClientConfig):
+        self._config = config
+        self._gql_client: AsyncAbstractGQLClient | None = None
+
+        self._lock = asyncio.Lock()
+        self._opened: int = 0
+        self._close_task = None
+
+    async def __aenter__(self):
+        async with self._lock:
+            self._opened += 1
+            if self._gql_client is None:
+                self._gql_client = self._config.configure()
+                await self._gql_client.__aenter__()
+            if self._close_task is not None:
+                self._close_task.cancel()
+                self._close_task = None
+        return self
+
+    async def __aexit__(self, exc_type=None, exc_val=None, exc_tb=None):
+        async with self._lock:
+            self._opened -= 1
+            if self._opened == 0 and self._close_task is None:
+                self._close_task = asyncio.create_task(self._delayed_close())
+
+    async def _delayed_close(self, delay: int = 10):
+        try:
+            await asyncio.sleep(delay)
+            async with self._lock:
+                if self._opened == 0 and self._gql_client is not None:
+                    await self._gql_client.__aexit__(None, None, None)
+                    self._gql_client = None
+        except asyncio.CancelledError:
+            pass
+
+    def validate(self, document: DocumentNode) -> bool:
+        return self._gql_client.validate(document)
+
+    async def execute(self, document: DocumentNode, operation_name: str, variables: dict | None = None, raise_on_timeout: bool = True):
+        try:
+            async with self:
+                return await self._gql_client.execute(document, operation_name=operation_name, variables=variables)
+
+        except Timeout as e:
+            logger.error('Timeout while query processing', exc_info=e,
+                         extra={'variables': str(variables), 'operation_name': operation_name})
+            if raise_on_timeout:
+                raise e
+        except Exception as e:
+            logger.error('Some exception was occured during query processing.', exc_info=e,
+                         extra={'variables': str(variables), 'operation_name': operation_name})
+
+            raise
+
+    async def paginate_items(
+            self,
+            document: DocumentNode,
+            operation_name: str,
+            variables: dict = None,
+            *,
+            page_size: int = 50,
+            extract_page: Callable[[dict], dict] = lambda d: d
+    ) -> AsyncIterator[dict]:
+        if variables is None:
+            variables = {}
+        total = 1  # some value greater than 0
+        offset = 0
+        while offset < total:
+            query_variables = {**variables, "offset": offset, "limit": page_size}
+            page = extract_page(await self.execute(document, operation_name, query_variables))
+            total = page['pagination']['total']
+            items = page['pagination'].get('list', tuple())
+            for item in items:
+                yield item
+            offset += len(items)
+
+    async def get_all_items(
+            self,
+            document: DocumentNode,
+            operation_name: str,
+            variables: dict = None,
+            *,
+            extract_page: Callable[[dict], dict] = lambda d: d
+    ):
+        if variables is None:
+            variables = {}
+        # get total only
+        query_variables = {**variables, "offset": 0, "limit": 0}
+        page = extract_page(await self.execute(document, operation_name, query_variables))
+        total = page['pagination']['total']
+
+        # get all items
+        query_variables = {**variables, "offset": 0, "limit": total}
+        page = extract_page(await self.execute(document, operation_name, query_variables))
+        items = page['pagination'].get('list', tuple())
+        for item in items:
+            yield item

talisman_api/api_client/composite.py

@@ -0,0 +1,29 @@
+from enum import Enum
+from typing import Iterator, Mapping
+
+from .client import TalismanAPIClient
+from .gql_clients import GQLClientConfig
+
+
+class APISchema(str, Enum):
+    PUBLIC = "public"
+    KB_UTILS = "kbutils"
+
+
+class CompositeTalismanAPIClient(Mapping[APISchema, TalismanAPIClient]):
+    def __init__(self, config: dict[str | APISchema, GQLClientConfig]):
+        self._api_clients: dict[APISchema, TalismanAPIClient] = {
+            APISchema(key): TalismanAPIClient(value) for key, value in config.items()
+        }
+
+    def __contains__(self, item: object) -> bool:
+        return item in self._api_clients
+
+    def __getitem__(self, item: APISchema) -> TalismanAPIClient:
+        return self._api_clients.get(item)
+
+    def __iter__(self) -> Iterator[APISchema]:
+        return iter(self._api_clients)
+
+    def __len__(self) -> int:
+        return len(self._api_clients)

talisman_api/api_client/gql_clients/__init__.py

@@ -0,0 +1,11 @@
+__all__ = [
+    'AsyncAbstractGQLClient',
+    'GQLClientConfig',
+    'AsyncKeycloakAwareGQLClient',
+    'AsyncNoAuthGQLClient'
+]
+
+from ._abstract import AsyncAbstractGQLClient
+from ._config import GQLClientConfig
+from ._keycloak import AsyncKeycloakAwareGQLClient
+from ._noauth import AsyncNoAuthGQLClient

talisman_api/api_client/gql_clients/_abstract.py

@@ -0,0 +1,90 @@
+import logging
+from abc import ABCMeta
+from asyncio import Semaphore
+from contextlib import AbstractAsyncContextManager
+from typing import Callable, Optional
+
+import backoff
+from aiorwlock import RWLock
+from gql import Client
+from gql.client import AsyncClientSession
+from gql.transport.aiohttp import AIOHTTPTransport, log as aiohttp_logger
+from gql.transport.exceptions import TransportQueryError
+from gql.transport.requests import log as requests_logger
+from graphql import DocumentNode
+
+from tp_interfaces.logging.time import AsyncTimeMeasurer
+
+requests_logger.setLevel(logging.WARNING)
+aiohttp_logger.setLevel(logging.ERROR)
+
+logger = logging.getLogger(__name__)
+
+
+class AsyncAbstractGQLClient(AbstractAsyncContextManager, metaclass=ABCMeta):
+
+    def __init__(self, gql_uri: str, timeout: int = 60, concurrency_limit: int = 10, retry_execute: bool | Callable = True):
+        self._gql_uri = gql_uri
+        self._timeout = timeout
+        self._retry_execute = retry_execute
+
+        self._client: Optional[Client] = None
+        self._session: Optional[AsyncClientSession] = None
+
+        self._sema = Semaphore(concurrency_limit)
+        self._rw_lock = RWLock()
+
+    def validate(self, document: DocumentNode) -> bool:
+        try:
+            self._client.validate(document)
+            return True
+        except Exception:
+            return False
+
+    async def execute(self, document: DocumentNode, variables=None, operation_name=None, extra_headers=None, timeout=None):
+        async with self._sema, self._rw_lock.reader_lock:
+            async with AsyncTimeMeasurer(
+                    f"query {operation_name}", inline_time=True, logger=logger, warning_threshold=5000,
+                    extra={"operation_name": operation_name, "variables": variables}
+            ):
+                return await self._session.execute(document, variables, operation_name)
+
+    async def _configure_session(self, headers: dict = None):
+        async with self._rw_lock.writer_lock:
+            await self._close_session()
+
+            transport = AIOHTTPTransport(url=self._gql_uri, headers=headers)
+            self._client = Client(transport=transport, fetch_schema_from_transport=True, execute_timeout=self._timeout)
+
+            # here we could change default behaviour of query retrying: just change retry_execute to backoff decorator
+            self._session = await self._client.connect_async(reconnecting=True, retry_execute=self._retry_execute)
+
+    async def _close_session(self):
+        if self._session is not None:
+            await self._client.close_async()
+        self._session = None
+        self._client = None
+
+
+def build_retry_execute(config: bool | dict = True) -> bool | Callable:
+    if isinstance(config, bool):
+        return config
+    if isinstance(config, dict):
+        if not config:
+            config = {
+                'wait_gen': 'expo',
+                'wait_gen_args': {
+                    'base': 2.71828,
+                    'factor': 2
+                }
+            }
+        wait_gen = getattr(backoff, config.get('wait_gen', 'expo'))
+        max_tries = config.get('max_tries', 5)
+        return backoff.on_exception(
+            wait_gen,
+            Exception,
+            max_tries=max_tries,
+            giveup=lambda e: isinstance(e, TransportQueryError),
+            **config.get('wait_gen_args', {})
+        )
+    raise ValueError

talisman_api/api_client/gql_clients/_config.py

@@ -0,0 +1,19 @@
+from typing import NamedTuple
+
+from ._abstract import AsyncAbstractGQLClient, build_retry_execute
+from ._keycloak import AsyncKeycloakAwareGQLClient
+from ._noauth import AsyncNoAuthGQLClient
+
+
+class GQLClientConfig(NamedTuple):
+    uri: str
+    auth: bool = False
+    timeout: int = 60
+    concurrency_limit: int = 30
+    retry_execute: bool | dict = True
+
+    def configure(self) -> AsyncAbstractGQLClient:
+        retry_execute = build_retry_execute(self.retry_execute)
+        if self.auth:
+            return AsyncKeycloakAwareGQLClient(self.uri, self.timeout, self.concurrency_limit, retry_execute)
+        return AsyncNoAuthGQLClient(self.uri, self.timeout, self.concurrency_limit, retry_execute)

talisman_api/api_client/gql_clients/_keycloak.py

@@ -0,0 +1,90 @@
+import enum
+import logging
+import os
+import time
+from asyncio import Lock
+from typing import Callable, Optional
+
+from keycloak import KeycloakOpenID
+from urllib3.util.retry import log
+
+from tp_interfaces.logging.time import TimeMeasurer
+from ._abstract import AsyncAbstractGQLClient
+
+logger = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
+
+
+class AuthEnvs(enum.Enum):
+    AUTH_URL = "KEYCLOAK_AUTH_URL"
+    REALM = "KEYCLOAK_REALM"
+    CLIENT_ID = "KEYCLOAK_CLIENT_ID"
+    CLIENT_KEY = "KEYCLOAK_CLIENT_KEY"
+    USER = "KEYCLOAK_USER"
+    PWD = "KEYCLOAK_PWD"
+
+    @property
+    def env(self) -> str:
+        return os.getenv(self.value)
+
+
+class AsyncKeycloakAwareGQLClient(AsyncAbstractGQLClient):
+    _TIME_OFFSET = 10  # in seconds
+
+    def __init__(self, gql_uri: str, timeout: int = 60, concurrency_limit: int = 10, retry_execute: bool | Callable = True):
+        super().__init__(gql_uri, timeout, concurrency_limit, retry_execute)
+
+        self._auth: dict[AuthEnvs, str] = {env: env.env for env in AuthEnvs}
+
+        not_set_envs = {env for env, val in self._auth.items() if val is None}
+        if not_set_envs:
+            raise ValueError(f"Authorization environment values are not set: {not_set_envs}")
+
+        self._keycloak_openid: Optional[KeycloakOpenID] = None
+        self._access_token: Optional[str] = None
+        self._refresh_token: Optional[str] = None
+        self._access_expiration_timestamp: float = 0
+        self._refresh_expiration_timestamp: float = 0
+
+        self._lock = Lock()
+
+    async def _ensure_session_liveness(self):
+        offsetted_time = time.time() + self._TIME_OFFSET
+        if offsetted_time < self._access_expiration_timestamp:
+            return
+
+        time_before_req = time.time()
+        if offsetted_time < self._refresh_expiration_timestamp and self._refresh_token is not None:
+            with TimeMeasurer("refreshing access token with refresh token", logger=logger):
+                token_info = self._keycloak_openid.refresh_token(self._refresh_token)
+        else:
+            with TimeMeasurer("refreshing access token with credentials", logger=logger):
+                token_info = self._keycloak_openid.token(self._auth[AuthEnvs.USER], self._auth[AuthEnvs.PWD])
+
+        self._access_token = token_info['access_token']
+        self._access_expiration_timestamp = time_before_req + token_info['expires_in']
+        self._refresh_token = token_info['refresh_token']
+        self._refresh_expiration_timestamp = time_before_req + token_info['refresh_expires_in']
+
+        headers = {"X-Auth-Token": self._access_token, "Authorization": f"Bearer {self._access_token}"}
+        await self._configure_session(headers)
+
+    async def __aenter__(self):
+        self._keycloak_openid = KeycloakOpenID(
+            self._auth[AuthEnvs.AUTH_URL], self._auth[AuthEnvs.REALM], self._auth[AuthEnvs.CLIENT_ID], self._auth[AuthEnvs.CLIENT_KEY]
+        )
+        return self
+
+    async def __aexit__(self, exc_type=None, exc_val=None, exc_tb=None):
+        async with self._lock:
+            async with self._rw_lock.writer_lock:
+                await self._close_session()
+
+            self._access_token, self._refresh_token = None, None
+            self._access_expiration_timestamp, self._refresh_expiration_timestamp = 0, 0
+            self._keycloak_openid = None
+
+    async def execute(self, document, variables=None, operation_name=None, extra_headers=None, timeout=None):
+        async with self._lock:  # lock session configuration to avoid concurrent keycloack requests
+            await self._ensure_session_liveness()
+        return await super().execute(document, variables, operation_name, extra_headers, timeout)

talisman_api/api_client/gql_clients/_noauth.py

@@ -0,0 +1,12 @@
+from ._abstract import AsyncAbstractGQLClient
+
+
+class AsyncNoAuthGQLClient(AsyncAbstractGQLClient):
+
+    async def __aenter__(self):
+        await self._configure_session()
+        return self
+
+    async def __aexit__(self, exc_type=None, exc_val=None, exc_tb=None):
+        async with self._rw_lock.writer_lock:
+            await self._close_session()

talisman_api/decorator.py

@@ -0,0 +1,9 @@
+from .abstract import AbstractTalismanAPI
+
+
+def version(v: str):
+    def decorate(cls: type[AbstractTalismanAPI]):
+        cls._VERSION = v
+        return cls
+
+    return decorate

talisman_api/wrapper.py

@@ -0,0 +1,54 @@
+from contextlib import AbstractAsyncContextManager
+from typing import AsyncIterator, Callable
+
+from graphql import DocumentNode
+
+from .api_client import TalismanAPIClient as BaseClient
+
+
+class TalismanAPIClient(AbstractAsyncContextManager):
+    def __init__(self, client: BaseClient, document: DocumentNode):
+        self._client = client
+        self._document = document
+
+    async def __aenter__(self):
+        await self._client.__aenter__()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self._client.__aexit__(exc_type, exc_val, exc_tb)
+
+    async def execute(self, operation_name: str, variables: dict | None = None, raise_on_timeout: bool = True):
+        return await self._client.execute(self._document, operation_name, variables, raise_on_timeout)
+
+    async def paginate_items(
+            self,
+            operation_name: str,
+            variables: dict = None,
+            *,
+            page_size: int = 20,
+            extract_page: Callable[[dict], dict] = lambda d: d
+    ) -> AsyncIterator[dict]:
+        async for i in self._client.paginate_items(
+                self._document,
+                operation_name,
+                variables,
+                page_size=page_size,
+                extract_page=extract_page
+        ):
+            yield i
+
+    async def get_all_items(
+            self,
+            operation_name: str,
+            variables: dict = None,
+            *,
+            extract_page: Callable[[dict], dict] = lambda d: d
+    ):
+        async for i in self._client.get_all_items(
+                self._document,
+                operation_name,
+                variables,
+                extract_page=extract_page
+        ):
+            yield i

talisman_api-1.0.0.data/data/VERSION

@@ -0,0 +1 @@
+1.0.0

talisman_api-1.0.0.dist-info/METADATA

@@ -0,0 +1,24 @@
+Metadata-Version: 2.1
+Name: talisman-api
+Version: 1.0.0
+Summary: Python Talisman API client for Talisman-based app
+Author: ISPRAS Talisman NLP team
+Author-email: modis@ispras.ru
+Maintainer: Vladimir Mayorov
+Maintainer-email: vmayorov@ispras.ru
+License: Apache Software License
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: License :: OSI Approved :: Apache Software License
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: talisman-interfaces (~=0.7)
+Requires-Dist: aiorwlock (~=1.3)
+Requires-Dist: gql[aiohttp] (~=3.4)
+Requires-Dist: requests (~=2.31)
+Requires-Dist: python-keycloak (~=2.16)
+Requires-Dist: typing-extensions (>=4.0.0)
+

talisman_api-1.0.0.dist-info/RECORD

@@ -0,0 +1,17 @@
+talisman_api/__init__.py,sha256=Xyz4wn-nSCuWn1ekRY61ym8-PhXglvqX4o9eveUOefc,302
+talisman_api/abstract.py,sha256=IUUbYOIGgRYafHJqct3JPwhknsBTynrC9LbKCgACKjo,3753
+talisman_api/decorator.py,sha256=otwEnX1AhPUvTblyr47Up1Xf1aHA_jQmGpzhsMXZygc,180
+talisman_api/wrapper.py,sha256=q9PMYJ6R9xOZOXGB2aOs4O0IxHCJH7SzYmI9yX1LAHg,1734
+talisman_api/api_client/__init__.py,sha256=3kLisxkvPOKBVhw9qwTZQmVjeJDmlO9IqgoaNrwRyxc,249
+talisman_api/api_client/client.py,sha256=4F83CU63T0lbN4td_yiaXok2UgDihGMxdf5EzapsC6g,4172
+talisman_api/api_client/composite.py,sha256=iKCGZP7gg2zP1UcNJYtm7FHblYloq9uqyFJzoTks3Y8,882
+talisman_api/api_client/gql_clients/__init__.py,sha256=3wGvo38C862_iOHQwu80Bl6c-OTIXDOBmhrVx9DAP08,306
+talisman_api/api_client/gql_clients/_abstract.py,sha256=OWctaZ3S6wcDIpc_gCeMOCFmVykFBCYAVGl43aUhZPw,3389
+talisman_api/api_client/gql_clients/_config.py,sha256=qnrB_yTwdPZqGCg2Bzfe5SkbOnQW72h8ddItjxmPvNs,703
+talisman_api/api_client/gql_clients/_keycloak.py,sha256=SruxIm_ripB8GVc_VWpPIprNyW7efxZMjvhwA-mksfA,3690
+talisman_api/api_client/gql_clients/_noauth.py,sha256=V9GDkMJg3NH5dZG3InWlpxv8sa1Eha220RYBHad_qao,353
+talisman_api-1.0.0.data/data/VERSION,sha256=klIfw8vZZL3J9YSpkbif3apXVO0cyW1tQkRTOGacEwU,5
+talisman_api-1.0.0.dist-info/METADATA,sha256=t1D--9EojEPZeqTxh2fabjCS1hBud2EeJjI7i1OEuUo,867
+talisman_api-1.0.0.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+talisman_api-1.0.0.dist-info/top_level.txt,sha256=9i4PbX8BUXG5HjLUv5hrPQg1vgyYBxhD4Uuf8jTVdw8,13
+talisman_api-1.0.0.dist-info/RECORD,,

talisman_api-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.40.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

talisman_api-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+talisman_api