python3-commons 0.0.0__py3-none-any.whl → 0.2.17__py3-none-any.whl

python3_commons/api_client.py

@@ -1,14 +1,16 @@
+import errno
 import logging
+from collections.abc import AsyncGenerator, Mapping, Sequence
 from contextlib import asynccontextmanager
 from datetime import UTC, datetime
 from enum import Enum
 from http import HTTPStatus
 from json import dumps
-from typing import AsyncGenerator, Literal, Mapping, Sequence
+from typing import Literal
 from uuid import uuid4
 
 from aiohttp import ClientResponse, ClientSession, ClientTimeout, client_exceptions
-from pydantic import HttpUrl
+from aiohttp.abc import URL
 
 from python3_commons import audit
 from python3_commons.conf import s3_settings
@@ -38,7 +40,7 @@ async def _store_response_for_audit(
 @asynccontextmanager
 async def request(
     client: ClientSession,
-    base_url: HttpUrl,
+    base_url: str,
     uri: str,
     query: Mapping | None = None,
     method: Literal['get', 'post', 'put', 'patch', 'options', 'head', 'delete'] = 'get',
@@ -52,18 +54,21 @@ async def request(
     date_path = now.strftime('%Y/%m/%d')
     timestamp = now.strftime('%H%M%S_%f')
     request_id = str(uuid4())[-12:]
-    uri_path = uri[:-1] if uri.endswith('/') else uri
-    uri_path = uri_path[1:] if uri_path.startswith('/') else uri_path
+    uri_path = uri.removesuffix('/')
+    uri_path = uri_path.removeprefix('/')
     url = f'{u[:-1] if (u := str(base_url)).endswith("/") else u}{uri}'
 
     if audit_name:
         curl_request = None
+        cookies = client.cookie_jar.filter_cookies(URL(base_url)) if base_url else None
 
         if method == 'get':
             if headers or query:
-                curl_request = request_to_curl(url, query, method, headers)
+                curl_request = request_to_curl(url=url, query=query, method=method, headers=headers, cookies=cookies)
         else:
-            curl_request = request_to_curl(url, query, method, headers, json, data)
+            curl_request = request_to_curl(
+                url=url, query=query, method=method, headers=headers, cookies=cookies, json=json, data=data
+            )
 
         if curl_request:
             await audit.write_audit_data(
@@ -71,6 +76,7 @@ async def request(
                 f'{date_path}/{audit_name}/{uri_path}/{method}_{timestamp}_{request_id}_request.txt',
                 curl_request.encode('utf-8'),
             )
+
     client_method = getattr(client, method)
 
     logger.debug(f'Requesting {method} {url}')
@@ -86,13 +92,25 @@ async def request(
                 else:
                     match response.status:
                         case HTTPStatus.UNAUTHORIZED:
-                            raise PermissionError('Unauthorized')
+                            msg = 'Unauthorized'
+
+                            raise PermissionError(msg)
                         case HTTPStatus.FORBIDDEN:
-                            raise PermissionError('Forbidden')
+                            msg = 'Forbidden'
+
+                            raise PermissionError(msg)
                         case HTTPStatus.NOT_FOUND:
-                            raise LookupError('Not found')
+                            msg = 'Not found'
+
+                            raise LookupError(msg)
                         case HTTPStatus.BAD_REQUEST:
-                            raise ValueError('Bad request')
+                            msg = 'Bad request'
+
+                            raise ValueError(msg)
+                        case HTTPStatus.TOO_MANY_REQUESTS:
+                            msg = 'Too many requests'
+
+                            raise InterruptedError(msg)
                         case _:
                             response.raise_for_status()
         else:
@@ -109,12 +127,22 @@ async def request(
                     await _store_response_for_audit(response, audit_name, uri_path, method, request_id)
 
                 yield response
+    except client_exceptions.ClientConnectorError as e:
+        msg = 'Cient connection error'
+
+        raise ConnectionRefusedError(msg) from e
     except client_exceptions.ClientOSError as e:
-        if e.errno == 32:
-            raise ConnectionResetError('Broken pipe') from e
-        elif e.errno == 104:
-            raise ConnectionResetError('Connection reset by peer') from e
+        if e.errno == errno.EPIPE:
+            msg = 'Broken pipe'
+
+            raise ConnectionResetError(msg) from e
+        elif e.errno == errno.ECONNRESET:
+            msg = 'Connection reset by peer'
+
+            raise ConnectionResetError(msg) from e
 
         raise
     except client_exceptions.ServerDisconnectedError as e:
-        raise ConnectionResetError('Server disconnected') from e
+        msg = 'Server disconnected'
+
+        raise ConnectionResetError(msg) from e

python3_commons/audit.py

@@ -1,167 +1,156 @@
 import asyncio
 import io
 import logging
-import tarfile
-from bz2 import BZ2Compressor
-from collections import deque
-from datetime import UTC, datetime, timedelta
-from typing import Generator, Iterable
+from datetime import UTC, datetime
 from uuid import uuid4
 
 from lxml import etree
-from minio import S3Error
 from zeep.plugins import Plugin
 from zeep.wsdl.definitions import AbstractOperation
 
 from python3_commons import object_storage
 from python3_commons.conf import S3Settings, s3_settings
-from python3_commons.object_storage import ObjectStorage
 
 logger = logging.getLogger(__name__)
 
 
-class GeneratedStream(io.BytesIO):
-    def __init__(self, generator: Generator[bytes, None, None], *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.generator = generator
-
-    def read(self, size: int = -1):
-        if size < 0:
-            while True:
-                try:
-                    chunk = next(self.generator)
-                except StopIteration:
-                    break
-                else:
-                    self.write(chunk)
-        else:
-            total_written_size = 0
-
-            while total_written_size < size:
-                try:
-                    chunk = next(self.generator)
-                except StopIteration:
-                    break
-                else:
-                    total_written_size += self.write(chunk)
-
-        self.seek(0)
-
-        if chunk := super().read(size):
-            pos = self.tell()
-
-            buf = self.getbuffer()
-            unread_data_size = len(buf) - pos
-
-            if unread_data_size > 0:
-                buf[:unread_data_size] = buf[pos : pos + unread_data_size]
-
-            del buf
-
-            self.seek(0)
-            self.truncate(unread_data_size)
-
-        return chunk
-
-    def readable(self):
-        return True
-
-
-def generate_archive(
-    objects: Iterable[tuple[str, datetime, bytes]], chunk_size: int = 4096
-) -> Generator[bytes, None, None]:
-    buffer = deque()
-
-    with tarfile.open(fileobj=buffer, mode='w') as archive:
-        for name, last_modified, content in objects:
-            logger.info(f'Adding {name} to archive')
-            info = tarfile.TarInfo(name)
-            info.size = len(content)
-            info.mtime = last_modified.timestamp()
-            archive.addfile(info, io.BytesIO(content))
-
-            buffer_length = buffer.tell()
-
-            while buffer_length >= chunk_size:
-                buffer.seek(0)
-                chunk = buffer.read(chunk_size)
-                chunk_len = len(chunk)
-
-                if not chunk:
-                    break
-
-                yield chunk
-
-                buffer.seek(0)
-                buffer.truncate(chunk_len)
-                buffer.seek(0, io.SEEK_END)
-                buffer_length = buffer.tell()
+# class GeneratedStream(io.BytesIO):
+#     def __init__(self, generator: AsyncGenerator[bytes], *args, **kwargs):
+#         super().__init__(*args, **kwargs)
+#         self.generator = generator
+#
+#     def read(self, size: int = -1):
+#         if size < 0:
+#             while True:
+#                 try:
+#                     chunk = anext(self.generator)
+#                 except StopIteration:
+#                     break
+#                 else:
+#                     self.write(chunk)
+#         else:
+#             total_written_size = 0
+#
+#             while total_written_size < size:
+#                 try:
+#                     chunk = anext(self.generator)
+#                 except StopIteration:
+#                     break
+#                 else:
+#                     total_written_size += self.write(chunk)
+#
+#         self.seek(0)
+#
+#         if chunk := super().read(size):
+#             pos = self.tell()
+#
+#             buf = self.getbuffer()
+#             unread_data_size = len(buf) - pos
+#
+#             if unread_data_size > 0:
+#                 buf[:unread_data_size] = buf[pos : pos + unread_data_size]
+#
+#             del buf
+#
+#             self.seek(0)
+#             self.truncate(unread_data_size)
+#
+#         return chunk
+#
+#     def readable(self):
+#         return True
+#
+#
+# async def generate_archive(
+#     objects: AsyncGenerator[tuple[str, datetime, bytes]], chunk_size: int = 4096
+# ) -> AsyncGenerator[bytes]:
+#     buffer = deque()
+#
+#     with tarfile.open(fileobj=buffer, mode='w') as archive:
+#         async for name, last_modified, content in objects:
+#             logger.info(f'Adding {name} to archive')
+#             info = tarfile.TarInfo(name)
+#             info.size = len(content)
+#             info.mtime = int(last_modified.timestamp())
+#             archive.addfile(info, io.BytesIO(content))
+#
+#             buffer_length = buffer.tell()
+#
+#             while buffer_length >= chunk_size:
+#                 buffer.seek(0)
+#                 chunk = buffer.read(chunk_size)
+#                 chunk_len = len(chunk)
+#
+#                 if not chunk:
+#                     break
+#
+#                 yield chunk
+#
+#                 buffer.seek(0)
+#                 buffer.truncate(chunk_len)
+#                 buffer.seek(0, io.SEEK_END)
+#                 buffer_length = buffer.tell()
+#
+#     while True:
+#         chunk = buffer.read(chunk_size)
+#
+#         if not chunk:
+#             break
+#
+#         yield chunk
+#
+#     buffer.seek(0)
+#     buffer.truncate(0)
+#
+#
+# async def generate_bzip2(chunks: AsyncGenerator[bytes]) -> AsyncGenerator[bytes]:
+#     compressor = BZ2Compressor()
+#
+#     async for chunk in chunks:
+#         if compressed_chunk := compressor.compress(chunk):
+#             yield compressed_chunk
+#
+#     if compressed_chunk := compressor.flush():
+#         yield compressed_chunk
+#
+
+# async def archive_audit_data(root_path: str = 'audit'):
+#     now = datetime.now(tz=UTC) - timedelta(days=1)
+#     year = now.year
+#     month = now.month
+#     day = now.day
+#     bucket_name = s3_settings.s3_bucket
+#     date_path = object_storage.get_absolute_path(f'{root_path}/{year}/{month:02}/{day:02}')
+#
+#     if objects := object_storage.get_objects(bucket_name, date_path, recursive=True):
+#         logger.info(f'Compacting files in: {date_path}')
+#
+#         generator = generate_archive(objects, chunk_size=900_000)
+#         bzip2_generator = generate_bzip2(generator)
+#         archive_stream = GeneratedStream(bzip2_generator)
+#
+#         archive_path = object_storage.get_absolute_path(f'audit/.archive/{year}_{month:02}_{day:02}.tar.bz2')
+#         await object_storage.put_object(bucket_name, archive_path, archive_stream, -1, part_size=5 * 1024 * 1024)
+#
+#         if errors := await object_storage.remove_objects(bucket_name, date_path):
+#             for error in errors:
+#                 logger.error(f'Failed to delete object in {bucket_name=}: {error}')
 
-    while True:
-        chunk = buffer.read(chunk_size)
 
-        if not chunk:
-            break
-
-        yield chunk
-
-    buffer.seek(0)
-    buffer.truncate(0)
-
-
-def generate_bzip2(chunks: Generator[bytes, None, None]) -> Generator[bytes, None, None]:
-    compressor = BZ2Compressor()
-
-    for chunk in chunks:
-        if compressed_chunk := compressor.compress(chunk):
-            yield compressed_chunk
-
-    if compressed_chunk := compressor.flush():
-        yield compressed_chunk
-
-
-def write_audit_data_sync(settings: S3Settings, key: str, data: bytes):
-    if settings.s3_secret_access_key:
+async def write_audit_data(settings: S3Settings, key: str, data: bytes):
+    if settings.aws_secret_access_key:
         try:
-            client = ObjectStorage(settings).get_client()
             absolute_path = object_storage.get_absolute_path(f'audit/{key}')
 
-            client.put_object(settings.s3_bucket, absolute_path, io.BytesIO(data), len(data))
-        except S3Error as e:
-            logger.error(f'Failed storing object in storage: {e}')
+            await object_storage.put_object(settings.s3_bucket, absolute_path, io.BytesIO(data), len(data))
+        except Exception:
+            logger.exception('Failed storing object in storage.')
         else:
             logger.debug(f'Stored object in storage: {key}')
     else:
         logger.debug(f'S3 is not configured, not storing object in storage: {key}')
 
 
-async def write_audit_data(settings: S3Settings, key: str, data: bytes):
-    await asyncio.to_thread(write_audit_data_sync, settings, key, data)
-
-
-async def archive_audit_data(root_path: str = 'audit'):
-    now = datetime.now(tz=UTC) - timedelta(days=1)
-    year = now.year
-    month = now.month
-    day = now.day
-    bucket_name = s3_settings.s3_bucket
-    date_path = object_storage.get_absolute_path(f'{root_path}/{year}/{month:02}/{day:02}')
-
-    if objects := object_storage.get_objects(bucket_name, date_path, recursive=True):
-        logger.info(f'Compacting files in: {date_path}')
-
-        generator = generate_archive(objects, chunk_size=900_000)
-        bzip2_generator = generate_bzip2(generator)
-        archive_stream = GeneratedStream(bzip2_generator)
-
-        archive_path = object_storage.get_absolute_path(f'audit/.archive/{year}_{month:02}_{day:02}.tar.bz2')
-        object_storage.put_object(bucket_name, archive_path, archive_stream, -1, part_size=5 * 1024 * 1024)
-
-        if errors := object_storage.remove_objects(bucket_name, date_path):
-            for error in errors:
-                logger.error(f'Failed to delete object in {bucket_name=}: {error}')
-
-
 class ZeepAuditPlugin(Plugin):
     def __init__(self, audit_name: str = 'zeep'):
         super().__init__()

python3_commons/auth.py

@@ -1,28 +1,28 @@
 import logging
+from collections.abc import Callable, Coroutine, MutableMapping, Sequence
 from http import HTTPStatus
-from typing import Annotated
+from typing import Annotated, Any, TypeVar
 
 import aiohttp
+import msgspec
 from fastapi import Depends, HTTPException
 from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
 from jose import JWTError, jwt
-from pydantic import BaseModel
 
 from python3_commons.conf import oidc_settings
 
 logger = logging.getLogger(__name__)
 
 
-class TokenData(BaseModel):
+class TokenData(msgspec.Struct):
     sub: str
-    aud: str
+    aud: str | Sequence[str]
     exp: int
     iss: str
 
 
+T = TypeVar('T', bound=TokenData)
 OIDC_CONFIG_URL = f'{oidc_settings.authority_url}/.well-known/openid-configuration'
-_JWKS: dict | None = None
-
 bearer_security = HTTPBearer(auto_error=oidc_settings.enabled)
 
 
@@ -30,53 +30,59 @@ async def fetch_openid_config() -> dict:
     """
     Fetch the OpenID configuration (including JWKS URI) from OIDC authority.
     """
-    async with aiohttp.ClientSession() as session:
-        async with session.get(OIDC_CONFIG_URL) as response:
-            if response.status != HTTPStatus.OK:
-                raise HTTPException(
-                    status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail='Failed to fetch OpenID configuration'
-                )
+    async with aiohttp.ClientSession() as session, session.get(OIDC_CONFIG_URL) as response:
+        if response.status != HTTPStatus.OK:
+            raise HTTPException(
+                status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail='Failed to fetch OpenID configuration'
+            )
 
-            return await response.json()
+        return await response.json()
 
 
 async def fetch_jwks(jwks_uri: str) -> dict:
     """
     Fetch the JSON Web Key Set (JWKS) for validating the token's signature.
     """
-    async with aiohttp.ClientSession() as session:
-        async with session.get(jwks_uri) as response:
-            if response.status != HTTPStatus.OK:
-                raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail='Failed to fetch JWKS')
-
-            return await response.json()
-
-
-async def get_verified_token(
-    authorization: Annotated[HTTPAuthorizationCredentials, Depends(bearer_security)],
-) -> TokenData | None:
-    """
-    Verify the JWT access token using OIDC authority JWKS.
-    """
-    global _JWKS
-
-    if not oidc_settings.enabled:
-        return None
-
-    token = authorization.credentials
-
-    try:
-        if not _JWKS:
-            openid_config = await fetch_openid_config()
-            _JWKS = await fetch_jwks(openid_config['jwks_uri'])
-
-        if oidc_settings.client_id:
-            payload = jwt.decode(token, _JWKS, algorithms=['RS256'], audience=oidc_settings.client_id)
-        else:
-            payload = jwt.decode(token, _JWKS, algorithms=['RS256'])
-
-        token_data = TokenData(**payload)
+    async with aiohttp.ClientSession() as session, session.get(jwks_uri) as response:
+        if response.status != HTTPStatus.OK:
+            raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail='Failed to fetch JWKS')
+
+        return await response.json()
+
+
+def get_token_verifier[T](
+    token_cls: type[T],
+    jwks: MutableMapping,
+) -> Callable[[HTTPAuthorizationCredentials], Coroutine[Any, Any, T | None]]:
+    async def get_verified_token(
+        authorization: Annotated[HTTPAuthorizationCredentials, Depends(bearer_security)],
+    ) -> T | None:
+        """
+        Verify the JWT access token using OIDC authority JWKS.
+        """
+        if not oidc_settings.enabled:
+            return None
+
+        token = authorization.credentials
+
+        try:
+            if not jwks:
+                openid_config = await fetch_openid_config()
+                _jwks = await fetch_jwks(openid_config['jwks_uri'])
+                jwks.clear()
+                jwks.update(_jwks)
+
+            if oidc_settings.client_id:
+                payload = jwt.decode(token, jwks, algorithms=['RS256'], audience=oidc_settings.client_id)
+            else:
+                payload = jwt.decode(token, jwks, algorithms=['RS256'])
+
+            token_data = token_cls(**payload)
+        except jwt.ExpiredSignatureError as e:
+            raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail='Token has expired') from e
+        except JWTError as e:
+            raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=f'Token is invalid: {e!s}') from e
 
         return token_data
-    except JWTError as e:
-        raise HTTPException(status_code=HTTPStatus.FORBIDDEN, detail=f'Token is invalid: {str(e)}')
+
+    return get_verified_token