python3-commons 0.0.0__py3-none-any.whl

python3_commons/__init__.py

@@ -0,0 +1,7 @@
+import importlib.metadata
+
+try:
+    dist_name = __name__
+    __version__ = importlib.metadata.version(dist_name)
+except importlib.metadata.PackageNotFoundError:
+    __version__ = 'unknown'

python3_commons/api_client.py

@@ -0,0 +1,120 @@
+import logging
+from contextlib import asynccontextmanager
+from datetime import UTC, datetime
+from enum import Enum
+from http import HTTPStatus
+from json import dumps
+from typing import AsyncGenerator, Literal, Mapping, Sequence
+from uuid import uuid4
+
+from aiohttp import ClientResponse, ClientSession, ClientTimeout, client_exceptions
+from pydantic import HttpUrl
+
+from python3_commons import audit
+from python3_commons.conf import s3_settings
+from python3_commons.helpers import request_to_curl
+from python3_commons.serializers.json import CustomJSONEncoder
+
+logger = logging.getLogger(__name__)
+
+
+async def _store_response_for_audit(
+    response: ClientResponse, audit_name: str, uri_path: str, method: str, request_id: str
+):
+    response_text = await response.text()
+
+    if response_text:
+        now = datetime.now(tz=UTC)
+        date_path = now.strftime('%Y/%m/%d')
+        timestamp = now.strftime('%H%M%S_%f')
+
+        await audit.write_audit_data(
+            s3_settings,
+            f'{date_path}/{audit_name}/{uri_path}/{method}_{timestamp}_{request_id}_response.txt',
+            response_text.encode('utf-8'),
+        )
+
+
+@asynccontextmanager
+async def request(
+    client: ClientSession,
+    base_url: HttpUrl,
+    uri: str,
+    query: Mapping | None = None,
+    method: Literal['get', 'post', 'put', 'patch', 'options', 'head', 'delete'] = 'get',
+    headers: Mapping | None = None,
+    json: Mapping | Sequence | str | None = None,
+    data: bytes | None = None,
+    timeout: ClientTimeout | Enum | None = None,
+    audit_name: str | None = None,
+) -> AsyncGenerator[ClientResponse]:
+    now = datetime.now(tz=UTC)
+    date_path = now.strftime('%Y/%m/%d')
+    timestamp = now.strftime('%H%M%S_%f')
+    request_id = str(uuid4())[-12:]
+    uri_path = uri[:-1] if uri.endswith('/') else uri
+    uri_path = uri_path[1:] if uri_path.startswith('/') else uri_path
+    url = f'{u[:-1] if (u := str(base_url)).endswith("/") else u}{uri}'
+
+    if audit_name:
+        curl_request = None
+
+        if method == 'get':
+            if headers or query:
+                curl_request = request_to_curl(url, query, method, headers)
+        else:
+            curl_request = request_to_curl(url, query, method, headers, json, data)
+
+        if curl_request:
+            await audit.write_audit_data(
+                s3_settings,
+                f'{date_path}/{audit_name}/{uri_path}/{method}_{timestamp}_{request_id}_request.txt',
+                curl_request.encode('utf-8'),
+            )
+    client_method = getattr(client, method)
+
+    logger.debug(f'Requesting {method} {url}')
+
+    try:
+        if method == 'get':
+            async with client_method(url, params=query, headers=headers, timeout=timeout) as response:
+                if audit_name:
+                    await _store_response_for_audit(response, audit_name, uri_path, method, request_id)
+
+                if response.ok:
+                    yield response
+                else:
+                    match response.status:
+                        case HTTPStatus.UNAUTHORIZED:
+                            raise PermissionError('Unauthorized')
+                        case HTTPStatus.FORBIDDEN:
+                            raise PermissionError('Forbidden')
+                        case HTTPStatus.NOT_FOUND:
+                            raise LookupError('Not found')
+                        case HTTPStatus.BAD_REQUEST:
+                            raise ValueError('Bad request')
+                        case _:
+                            response.raise_for_status()
+        else:
+            if json:
+                data = dumps(json, cls=CustomJSONEncoder).encode('utf-8')
+
+                if headers:
+                    headers = {**headers, 'Content-Type': 'application/json'}
+                else:
+                    headers = {'Content-Type': 'application/json'}
+
+            async with client_method(url, params=query, data=data, headers=headers, timeout=timeout) as response:
+                if audit_name:
+                    await _store_response_for_audit(response, audit_name, uri_path, method, request_id)
+
+                yield response
+    except client_exceptions.ClientOSError as e:
+        if e.errno == 32:
+            raise ConnectionResetError('Broken pipe') from e
+        elif e.errno == 104:
+            raise ConnectionResetError('Connection reset by peer') from e
+
+        raise
+    except client_exceptions.ServerDisconnectedError as e:
+        raise ConnectionResetError('Server disconnected') from e

python3_commons/audit.py

@@ -0,0 +1,196 @@
+import asyncio
+import io
+import logging
+import tarfile
+from bz2 import BZ2Compressor
+from collections import deque
+from datetime import UTC, datetime, timedelta
+from typing import Generator, Iterable
+from uuid import uuid4
+
+from lxml import etree
+from minio import S3Error
+from zeep.plugins import Plugin
+from zeep.wsdl.definitions import AbstractOperation
+
+from python3_commons import object_storage
+from python3_commons.conf import S3Settings, s3_settings
+from python3_commons.object_storage import ObjectStorage
+
+logger = logging.getLogger(__name__)
+
+
+class GeneratedStream(io.BytesIO):
+    def __init__(self, generator: Generator[bytes, None, None], *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.generator = generator
+
+    def read(self, size: int = -1):
+        if size < 0:
+            while True:
+                try:
+                    chunk = next(self.generator)
+                except StopIteration:
+                    break
+                else:
+                    self.write(chunk)
+        else:
+            total_written_size = 0
+
+            while total_written_size < size:
+                try:
+                    chunk = next(self.generator)
+                except StopIteration:
+                    break
+                else:
+                    total_written_size += self.write(chunk)
+
+        self.seek(0)
+
+        if chunk := super().read(size):
+            pos = self.tell()
+
+            buf = self.getbuffer()
+            unread_data_size = len(buf) - pos
+
+            if unread_data_size > 0:
+                buf[:unread_data_size] = buf[pos : pos + unread_data_size]
+
+            del buf
+
+            self.seek(0)
+            self.truncate(unread_data_size)
+
+        return chunk
+
+    def readable(self):
+        return True
+
+
+def generate_archive(
+    objects: Iterable[tuple[str, datetime, bytes]], chunk_size: int = 4096
+) -> Generator[bytes, None, None]:
+    buffer = deque()
+
+    with tarfile.open(fileobj=buffer, mode='w') as archive:
+        for name, last_modified, content in objects:
+            logger.info(f'Adding {name} to archive')
+            info = tarfile.TarInfo(name)
+            info.size = len(content)
+            info.mtime = last_modified.timestamp()
+            archive.addfile(info, io.BytesIO(content))
+
+            buffer_length = buffer.tell()
+
+            while buffer_length >= chunk_size:
+                buffer.seek(0)
+                chunk = buffer.read(chunk_size)
+                chunk_len = len(chunk)
+
+                if not chunk:
+                    break
+
+                yield chunk
+
+                buffer.seek(0)
+                buffer.truncate(chunk_len)
+                buffer.seek(0, io.SEEK_END)
+                buffer_length = buffer.tell()
+
+    while True:
+        chunk = buffer.read(chunk_size)
+
+        if not chunk:
+            break
+
+        yield chunk
+
+    buffer.seek(0)
+    buffer.truncate(0)
+
+
+def generate_bzip2(chunks: Generator[bytes, None, None]) -> Generator[bytes, None, None]:
+    compressor = BZ2Compressor()
+
+    for chunk in chunks:
+        if compressed_chunk := compressor.compress(chunk):
+            yield compressed_chunk
+
+    if compressed_chunk := compressor.flush():
+        yield compressed_chunk
+
+
+def write_audit_data_sync(settings: S3Settings, key: str, data: bytes):
+    if settings.s3_secret_access_key:
+        try:
+            client = ObjectStorage(settings).get_client()
+            absolute_path = object_storage.get_absolute_path(f'audit/{key}')
+
+            client.put_object(settings.s3_bucket, absolute_path, io.BytesIO(data), len(data))
+        except S3Error as e:
+            logger.error(f'Failed storing object in storage: {e}')
+        else:
+            logger.debug(f'Stored object in storage: {key}')
+    else:
+        logger.debug(f'S3 is not configured, not storing object in storage: {key}')
+
+
+async def write_audit_data(settings: S3Settings, key: str, data: bytes):
+    await asyncio.to_thread(write_audit_data_sync, settings, key, data)
+
+
+async def archive_audit_data(root_path: str = 'audit'):
+    now = datetime.now(tz=UTC) - timedelta(days=1)
+    year = now.year
+    month = now.month
+    day = now.day
+    bucket_name = s3_settings.s3_bucket
+    date_path = object_storage.get_absolute_path(f'{root_path}/{year}/{month:02}/{day:02}')
+
+    if objects := object_storage.get_objects(bucket_name, date_path, recursive=True):
+        logger.info(f'Compacting files in: {date_path}')
+
+        generator = generate_archive(objects, chunk_size=900_000)
+        bzip2_generator = generate_bzip2(generator)
+        archive_stream = GeneratedStream(bzip2_generator)
+
+        archive_path = object_storage.get_absolute_path(f'audit/.archive/{year}_{month:02}_{day:02}.tar.bz2')
+        object_storage.put_object(bucket_name, archive_path, archive_stream, -1, part_size=5 * 1024 * 1024)
+
+        if errors := object_storage.remove_objects(bucket_name, date_path):
+            for error in errors:
+                logger.error(f'Failed to delete object in {bucket_name=}: {error}')
+
+
+class ZeepAuditPlugin(Plugin):
+    def __init__(self, audit_name: str = 'zeep'):
+        super().__init__()
+        self.audit_name = audit_name
+
+    def store_audit_in_s3(self, envelope, operation: AbstractOperation, direction: str):
+        xml = etree.tostring(envelope, encoding='UTF-8', pretty_print=True)
+        now = datetime.now(tz=UTC)
+        date_path = now.strftime('%Y/%m/%d')
+        timestamp = now.strftime('%H%M%S')
+        path = f'{date_path}/{self.audit_name}/{operation.name}/{timestamp}_{str(uuid4())[-12:]}_{direction}.xml'
+        coro = write_audit_data(s3_settings, path, xml)
+
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            loop = None
+
+        if loop and loop.is_running():
+            loop.create_task(coro)
+        else:
+            asyncio.run(coro)
+
+    def ingress(self, envelope, http_headers, operation: AbstractOperation):
+        self.store_audit_in_s3(envelope, operation, 'ingress')
+
+        return envelope, http_headers
+
+    def egress(self, envelope, http_headers, operation: AbstractOperation, binding_options):
+        self.store_audit_in_s3(envelope, operation, 'egress')
+
+        return envelope, http_headers

python3_commons/auth.py

@@ -0,0 +1,82 @@
+import logging
+from http import HTTPStatus
+from typing import Annotated
+
+import aiohttp
+from fastapi import Depends, HTTPException
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+from jose import JWTError, jwt
+from pydantic import BaseModel
+
+from python3_commons.conf import oidc_settings
+
+logger = logging.getLogger(__name__)
+
+
+class TokenData(BaseModel):
+    sub: str
+    aud: str
+    exp: int
+    iss: str
+
+
+OIDC_CONFIG_URL = f'{oidc_settings.authority_url}/.well-known/openid-configuration'
+_JWKS: dict | None = None
+
+bearer_security = HTTPBearer(auto_error=oidc_settings.enabled)
+
+
+async def fetch_openid_config() -> dict:
+    """
+    Fetch the OpenID configuration (including JWKS URI) from OIDC authority.
+    """
+    async with aiohttp.ClientSession() as session:
+        async with session.get(OIDC_CONFIG_URL) as response:
+            if response.status != HTTPStatus.OK:
+                raise HTTPException(
+                    status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail='Failed to fetch OpenID configuration'
+                )
+
+            return await response.json()
+
+
+async def fetch_jwks(jwks_uri: str) -> dict:
+    """
+    Fetch the JSON Web Key Set (JWKS) for validating the token's signature.
+    """
+    async with aiohttp.ClientSession() as session:
+        async with session.get(jwks_uri) as response:
+            if response.status != HTTPStatus.OK:
+                raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail='Failed to fetch JWKS')
+
+            return await response.json()
+
+
+async def get_verified_token(
+    authorization: Annotated[HTTPAuthorizationCredentials, Depends(bearer_security)],
+) -> TokenData | None:
+    """
+    Verify the JWT access token using OIDC authority JWKS.
+    """
+    global _JWKS
+
+    if not oidc_settings.enabled:
+        return None
+
+    token = authorization.credentials
+
+    try:
+        if not _JWKS:
+            openid_config = await fetch_openid_config()
+            _JWKS = await fetch_jwks(openid_config['jwks_uri'])
+
+        if oidc_settings.client_id:
+            payload = jwt.decode(token, _JWKS, algorithms=['RS256'], audience=oidc_settings.client_id)
+        else:
+            payload = jwt.decode(token, _JWKS, algorithms=['RS256'])
+
+        token_data = TokenData(**payload)
+
+        return token_data
+    except JWTError as e:
+        raise HTTPException(status_code=HTTPStatus.FORBIDDEN, detail=f'Token is invalid: {str(e)}')

python3_commons/cache.py

@@ -0,0 +1,261 @@
+import logging
+import socket
+from platform import platform
+from typing import Any, Mapping, Sequence
+
+import valkey
+from pydantic import RedisDsn
+from valkey.asyncio import ConnectionPool, Sentinel, StrictValkey, Valkey
+from valkey.asyncio.retry import Retry
+from valkey.backoff import FullJitterBackoff
+from valkey.typing import ResponseT
+
+from python3_commons.conf import valkey_settings
+from python3_commons.helpers import SingletonMeta
+from python3_commons.serializers.msgspec import (
+    deserialize_msgpack,
+    deserialize_msgpack_native,
+    serialize_msgpack,
+    serialize_msgpack_native,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class AsyncValkeyClient(metaclass=SingletonMeta):
+    def __init__(self, dsn: RedisDsn, sentinel_dsn: RedisDsn | None):
+        self._valkey_pool = None
+        self._valkey = None
+
+        if sentinel_dsn:
+            self._initialize_sentinel(sentinel_dsn)
+        else:
+            self._initialize_standard_pool(dsn)
+
+    @staticmethod
+    def _get_keepalive_options():
+        if platform == 'linux' or platform == 'darwin':
+            return {socket.TCP_KEEPIDLE: 10, socket.TCP_KEEPINTVL: 5, socket.TCP_KEEPCNT: 5}
+        else:
+            return {}
+
+    def _initialize_sentinel(self, dsn: RedisDsn):
+        sentinel = Sentinel(
+            [(dsn.host, dsn.port)],
+            socket_connect_timeout=10,
+            socket_timeout=60,
+            password=dsn.password,
+            sentinel_kwargs={'password': dsn.password},
+        )
+
+        ka_options = self._get_keepalive_options()
+
+        self._valkey = sentinel.master_for(
+            'myprimary',
+            valkey_class=StrictValkey,
+            socket_connect_timeout=10,
+            socket_timeout=60,
+            health_check_interval=30,
+            retry_on_timeout=True,
+            retry=Retry(FullJitterBackoff(cap=5, base=1), 5),
+            socket_keepalive=True,
+            socket_keepalive_options=ka_options,
+        )
+
+    def _initialize_standard_pool(self, dsn: RedisDsn):
+        self._valkey_pool = ConnectionPool.from_url(str(dsn))
+        self._valkey = StrictValkey(connection_pool=self._valkey_pool)
+
+    def get_client(self) -> Valkey:
+        return self._valkey
+
+
+def get_valkey_client() -> Valkey:
+    return AsyncValkeyClient(valkey_settings.dsn, valkey_settings.sentinel_dsn).get_client()
+
+
+async def scan(
+    cursor: int = 0,
+    match: bytes | str | memoryview | None = None,
+    count: int | None = None,
+    _type: str | None = None,
+    **kwargs,
+) -> ResponseT:
+    return await get_valkey_client().scan(cursor, match, count, _type, **kwargs)
+
+
+async def delete(*names: str | bytes | memoryview):
+    await get_valkey_client().delete(*names)
+
+
+async def store_bytes(name: str, data: bytes, ttl: int = None, if_not_set: bool = False):
+    r = get_valkey_client()
+
+    return await r.set(name, data, ex=ttl, nx=if_not_set)
+
+
+async def get_bytes(name: str) -> bytes | None:
+    r = get_valkey_client()
+
+    return await r.get(name)
+
+
+async def store(name: str, obj: Any, ttl: int = None, if_not_set: bool = False):
+    return await store_bytes(name, serialize_msgpack_native(obj), ttl, if_not_set)
+
+
+async def get(name: str, default=None, data_type: Any = None) -> Any:
+    if data := await get_bytes(name):
+        return deserialize_msgpack_native(data, data_type)
+
+    return default
+
+
+async def store_string(name: str, data: str, ttl: int = None):
+    await store_bytes(name, data.encode(), ttl)
+
+
+async def get_string(name: str) -> str | None:
+    if data := await get_bytes(name):
+        return data.decode('utf-8')
+
+    return None
+
+
+async def store_sequence(name: str, data: Sequence, ttl: int = None):
+    if data:
+        try:
+            r = get_valkey_client()
+            await r.rpush(name, *map(serialize_msgpack_native, data))
+
+            if ttl:
+                await r.expire(name, ttl)
+        except valkey.exceptions.ConnectionError as e:
+            logger.error(f'Failed to store sequence in cache: {e}')
+
+
+async def get_sequence(name: str, _type: type = list) -> Sequence:
+    r = get_valkey_client()
+    lrange = await r.lrange(name, 0, -1)
+
+    return _type(map(deserialize_msgpack_native, lrange))
+
+
+async def store_dict(name: str, data: Mapping, ttl: int = None):
+    if data:
+        try:
+            r = get_valkey_client()
+            data = {k: serialize_msgpack_native(v) for k, v in data.items()}
+            await r.hset(name, mapping=data)
+
+            if ttl:
+                await r.expire(name, ttl)
+        except valkey.exceptions.ConnectionError as e:
+            logger.error(f'Failed to store dict in cache: {e}')
+
+
+async def get_dict(name: str, value_data_type=None) -> dict | None:
+    r = get_valkey_client()
+
+    if data := await r.hgetall(name):
+        data = {k.decode(): deserialize_msgpack(v, value_data_type) for k, v in data.items()}
+
+        return data
+
+    return None
+
+
+async def set_dict(name: str, mapping: dict, ttl: int = None):
+    if mapping:
+        try:
+            r = get_valkey_client()
+            mapping = {str(k): serialize_msgpack(v) for k, v in mapping.items()}
+            await r.hset(name, mapping=mapping)
+
+            if ttl:
+                await r.expire(name, ttl)
+        except valkey.exceptions.ConnectionError as e:
+            logger.error(f'Failed to set dict in cache: {e}')
+
+
+async def get_dict_item(name: str, key: str, data_type=None, default=None):
+    try:
+        r = get_valkey_client()
+
+        if data := await r.hget(name, key):
+            return deserialize_msgpack_native(data, data_type)
+
+        return default
+    except valkey.exceptions.ConnectionError as e:
+        logger.error(f'Failed to get dict item from cache: {e}')
+
+    return None
+
+
+async def set_dict_item(name: str, key: str, obj: Any):
+    try:
+        r = get_valkey_client()
+        await r.hset(name, key, serialize_msgpack_native(obj))
+    except valkey.exceptions.ConnectionError as e:
+        logger.error(f'Failed to set dict item in cache: {e}')
+
+
+async def delete_dict_item(name: str, *keys):
+    try:
+        r = get_valkey_client()
+        await r.hdel(name, *keys)
+    except valkey.exceptions.ConnectionError as e:
+        logger.error(f'Failed to delete dict item from cache: {e}')
+
+
+async def store_set(name: str, value: set, ttl: int = None):
+    try:
+        r = get_valkey_client()
+        await r.sadd(name, *map(serialize_msgpack_native, value))
+
+        if ttl:
+            await r.expire(name, ttl)
+    except valkey.exceptions.ConnectionError as e:
+        logger.error(f'Failed to store set in cache: {e}')
+
+
+async def has_set_item(name: str, value: str) -> bool:
+    try:
+        r = get_valkey_client()
+
+        return await r.sismember(name, serialize_msgpack_native(value)) == 1
+    except valkey.exceptions.ConnectionError as e:
+        logger.error(f'Failed to check if set has item in cache: {e}')
+
+    return False
+
+
+async def add_set_item(name: str, *values: str):
+    try:
+        r = get_valkey_client()
+        await r.sadd(name, *map(serialize_msgpack_native, values))
+    except valkey.exceptions.ConnectionError as e:
+        logger.error(f'Failed to add set item into cache: {e}')
+
+
+async def delete_set_item(name: str, value: str):
+    r = get_valkey_client()
+    await r.srem(name, serialize_msgpack_native(value))
+
+
+async def get_set_members(name: str) -> set[str] | None:
+    try:
+        r = get_valkey_client()
+        smembers = await r.smembers(name)
+
+        return set(map(deserialize_msgpack_native, smembers))
+    except valkey.exceptions.ConnectionError as e:
+        logger.error(f'Failed to get set members from cache: {e}')
+
+    return None
+
+
+async def exists(name: str) -> bool:
+    r = get_valkey_client()
+
+    return await r.exists(name) == 1