python3-commons 0.3.19__py2.py3-none-any.whl → 0.5.0__py2.py3-none-any.whl

python3_commons/audit.py

@@ -0,0 +1,102 @@
+import asyncio
+import logging
+import tarfile
+from datetime import date, datetime, timedelta, UTC
+from io import BytesIO
+from uuid import uuid4
+from zoneinfo import ZoneInfo
+
+from lxml import etree
+from zeep.plugins import Plugin
+from zeep.wsdl.definitions import AbstractOperation
+
+from python3_commons import object_storage
+from python3_commons.conf import s3_settings
+
+logger = logging.getLogger(__name__)
+
+
+class ZeepAuditPlugin(Plugin):
+    def __init__(self, audit_name: str = 'zeep'):
+        super().__init__()
+        self.audit_name = audit_name
+
+    @staticmethod
+    def store_audit_in_s3(envelope, operation: AbstractOperation, direction: str):
+        xml = etree.tostring(envelope, encoding='UTF-8', pretty_print=True)
+        now = datetime.now(tz=UTC)
+        date_path = now.strftime('%Y/%m/%d')
+        timestamp = now.strftime('%H%M%S')
+        coro = object_storage.store_bytes_in_s3(
+            settings, xml,
+            f'audit/{date_path}/{self.audit_name}/{operation.name}/{timestamp}_{str(uuid4())[-12:]}_{direction}.xml'
+        )
+
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            loop = None
+
+        if loop and loop.is_running():
+            loop.create_task(coro)
+        else:
+            asyncio.run(coro)
+
+    def ingress(self, envelope, http_headers, operation: AbstractOperation):
+        self.store_audit_in_s3(envelope, operation, 'ingress')
+
+        return envelope, http_headers
+
+    def egress(self, envelope, http_headers, operation: AbstractOperation, binding_options):
+        self.store_audit_in_s3(envelope, operation, 'egress')
+
+        return envelope, http_headers
+
+
+async def write_audit_data(settings: S3Settings, key: str, data: bytes):
+    if settings.s3_secret_access_key:
+        try:
+            client = get_s3_client(settings)
+
+            client.put_object(settings.s3_bucket, key, io.BytesIO(data), len(data))
+        except S3Error as e:
+            logger.error(f'Failed storing object in storage: {e}')
+        else:
+            logger.debug(f'Stored object in storage: {key}')
+    else:
+        logger.debug(f'S3 is not configured, not storing object in storage: {key}')
+
+
+async def archive_audit_data(root_path: str = 'audit'):
+    now = datetime.now(tz=UTC) - timedelta(days=1)
+    year = now.year
+    month = now.month
+    day = now.day
+    bucket_name = s3_settings.s3_bucket
+    fo = BytesIO()
+    object_names = []
+    date_path = object_storage.get_absolute_path(f'{root_path}/{year}/{month:02}/{day:02}')
+
+    with tarfile.open(fileobj=fo, mode='w|bz2') as archive:
+        if objects := object_storage.get_objects(bucket_name, date_path, recursive=True):
+            logger.info(f'Compacting files in: {date_path}')
+
+            for name, last_modified, content in objects:
+                info = tarfile.TarInfo(name)
+                info.size = len(content)
+                info.mtime = last_modified.timestamp()
+                archive.addfile(info, BytesIO(content))
+                object_names.append(name)
+
+    fo.seek(0)
+
+    if object_names:
+        archive_path = object_storage.get_absolute_path(
+            f'.archive/{year}_{month:02}/{year}_{month:02}_{day:02}.tar.bz2')
+        object_storage.put_object(bucket_name, archive_path, fo, fo.getbuffer().nbytes)
+
+        if errors := object_storage.remove_objects(bucket_name, object_names=object_names):
+            for error in errors:
+                logger.error(f'Failed to delete object in {bucket_name=}: {error}')
+    else:
+        logger.info('No objects to archive found.')

python3_commons/conf.py

@@ -15,6 +15,7 @@ class S3Settings(BaseSettings):
     s3_secure: bool = True
     s3_bucket: str | None = None
     s3_bucket_root: str | None = None
+    s3_cert_verify: bool = True
 
 
 settings = CommonSettings()

python3_commons/object_storage.py

@@ -1,29 +1,29 @@
+import io
 import logging
 from datetime import datetime
-from io import IOBase
-from typing import Generator, Iterable, Any
+from typing import Generator, Iterable
 
-from minio import Minio
+from minio import Minio, S3Error
 from minio.datatypes import Object
-from minio.deleteobjects import DeleteObject
+from minio.deleteobjects import DeleteObject, DeleteError
 
-from python3_commons import minio
-from python3_commons.conf import s3_settings
+from python3_commons.conf import s3_settings, S3Settings
 
 logger = logging.getLogger(__name__)
 __CLIENT = None
 
 
-def get_s3_client() -> Minio:
+def get_s3_client(settings: S3Settings) -> Minio:
     global __CLIENT
 
     if not __CLIENT and s3_settings.s3_endpoint_url:
-        __CLIENT = minio.get_client(
-            s3_settings.s3_endpoint_url,
-            s3_settings.s3_region_name,
-            s3_settings.s3_access_key_id,
-            s3_settings.s3_secret_access_key,
-            s3_settings.s3_secure
+        __CLIENT = Minio(
+            settings.s3_endpoint_url,
+            region=settings.s3_region_name,
+            access_key=settings.s3_access_key_id.get_secret_value(),
+            secret_key=settings.s3_secret_access_key.get_secret_value(),
+            secure=settings.s3_secure,
+            cert_check=settings.s3_cert_verify
         )
 
     return __CLIENT
@@ -34,13 +34,13 @@ def get_absolute_path(path: str) -> str:
         path = path[1:]
 
     if bucket_root := s3_settings.s3_bucket_root:
-        path = f'{bucket_root[:1] if bucket_root.startswith("/") else bucket_root}/{path}'
+        path = f'{bucket_root[:1] if bucket_root.startswith('/') else bucket_root}/{path}'
 
     return path
 
 
-def put_object(bucket_name: str, path: str, data: IOBase, length: int) -> str:
-    s3_client = get_s3_client()
+def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int) -> str:
+    s3_client = get_s3_client(s3_settings)
 
     if s3_client:
         result = s3_client.put_object(bucket_name, path, data, length)
@@ -53,7 +53,7 @@ def put_object(bucket_name: str, path: str, data: IOBase, length: int) -> str:
 
 
 def get_object_stream(bucket_name: str, path: str):
-    s3_client = get_s3_client()
+    s3_client = get_s3_client(s3_settings)
 
     if s3_client:
         logger.debug(f'Getting object from object storage: {bucket_name}:{path}')
@@ -85,7 +85,7 @@ def get_object(bucket_name: str, path: str) -> bytes:
 
 
 def list_objects(bucket_name: str, prefix: str, recursive: bool = True) -> Generator[Object, None, None]:
-    s3_client = get_s3_client()
+    s3_client = get_s3_client(s3_settings)
 
     yield from s3_client.list_objects(bucket_name, prefix=prefix, recursive=recursive)
 
@@ -104,13 +104,13 @@ def get_objects(bucket_name: str, path: str,
 
 
 def remove_object(bucket_name: str, object_name: str):
-    s3_client = get_s3_client()
+    s3_client = get_s3_client(s3_settings)
     s3_client.remove_object(bucket_name, object_name)
 
 
 def remove_objects(bucket_name: str, prefix: str = None,
-                   object_names: Iterable[str] = None) -> Generator[Any, Any, None] | None:
-    s3_client = get_s3_client()
+                   object_names: Iterable[str] = None) -> Iterable[DeleteError] | None:
+    s3_client = get_s3_client(s3_settings)
 
     if prefix:
         delete_object_list = map(

{python3_commons-0.3.19.dist-info → python3_commons-0.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python3-commons
-Version: 0.3.19
+Version: 0.5.0
 Summary: Re-usable Python3 code
 Home-page: https://github.com/kamikaze/python3-commons
 Author: Oleg Korsak
@@ -15,11 +15,13 @@ Description-Content-Type: text/x-rst; charset=UTF-8
 License-File: LICENSE
 License-File: AUTHORS.rst
 Requires-Dist: asyncpg ==0.29.0
+Requires-Dist: lxml ==5.2.2
 Requires-Dist: minio ==7.2.7
 Requires-Dist: msgpack ==1.0.8
 Requires-Dist: msgspec ==0.18.6
-Requires-Dist: pydantic[email] ==2.7.1
+Requires-Dist: pydantic[email] ==2.7.2
 Requires-Dist: pydantic-settings ==2.2.1
+Requires-Dist: zeep ==4.2.1
 Provides-Extra: testing
 Requires-Dist: pytest ; extra == 'testing'
 Requires-Dist: pytest-cov ; extra == 'testing'

{python3_commons-0.3.19.dist-info → python3_commons-0.5.0.dist-info}/RECORD

@@ -1,10 +1,10 @@
 python3_commons/__init__.py,sha256=h-KTJUaQ50E3RmkTn_GO88IRunmDTEpNc3ylpFvCTOc,339
-python3_commons/conf.py,sha256=yAsgMV264NXA3mktvP9C3WoZQJiuw2nuVgLigqUwaZs,566
+python3_commons/audit.py,sha256=n63-x3Segw7PjYvDak5RefMxQ8uC_IxtzmQr1L-VTaw,3571
+python3_commons/conf.py,sha256=vSXyFwXx2wb1uy8IffeeI-RoTqhUZs0RLSSG2OLc2ss,598
 python3_commons/db.py,sha256=qhaDIdzBWgFyeP_XPKfHZlYVlwS2bpBPYMv84yV6820,738
 python3_commons/fs.py,sha256=wfLjybXndwLqNlOxTpm_HRJnuTcC4wbrHEOaEeCo9Wc,337
 python3_commons/helpers.py,sha256=wI7afc-8o-SBpLBHQnWXiudw456EAVKguJaeZiSNXwE,1377
-python3_commons/minio.py,sha256=0WzxmfKEahxjlC2EdMC8mqy1UZTe2uv9bj1US_IqIoQ,819
-python3_commons/object_storage.py,sha256=ka2EIPRNyFx1N2evr7VZPCOaroXzbgYJAUIPlMTuoNY,3644
+python3_commons/object_storage.py,sha256=xQy8Jzb0_nS6vxQ6RyNA5FV_ql3k88iZQ5RPWJJiBUo,3798
 python3_commons/logging/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 python3_commons/logging/filters.py,sha256=fuyjXZAUm-i2MNrxvFYag8F8Rr27x8W8MdV3ke6miSs,175
 python3_commons/logging/formatter.py,sha256=UXmmh1yd5Kc2dpvSHn6uCWLDWE2LMjlYAaH8cg3siV4,720
@@ -12,9 +12,9 @@ python3_commons/serializers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 python3_commons/serializers/json.py,sha256=P288wWz9ic38QWEMrpp_uwKPYkQiOgvE1cI4WZn6ZCg,808
 python3_commons/serializers/msgpack.py,sha256=P7CZoRTBeDtgALT5GTOZVCNM_3snOfCnfh5S3zcCIEY,1679
 python3_commons/serializers/msgspec.py,sha256=ZrfQWBz_67t3yjU_S6avnRJZzYN-HFzef2qGB5W21KI,2023
-python3_commons-0.3.19.dist-info/AUTHORS.rst,sha256=3R9JnfjfjH5RoPWOeqKFJgxVShSSfzQPIrEr1nxIo9Q,90
-python3_commons-0.3.19.dist-info/LICENSE,sha256=xxILuojHm4fKQOrMHPSslbyy6WuKAN2RiG74HbrYfzM,34575
-python3_commons-0.3.19.dist-info/METADATA,sha256=mKBphqIK9O2vVBaEU66Y4X3D6__tpHLOgXk-FRa-530,919
-python3_commons-0.3.19.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
-python3_commons-0.3.19.dist-info/top_level.txt,sha256=lJI6sCBf68eUHzupCnn2dzG10lH3jJKTWM_hrN1cQ7M,16
-python3_commons-0.3.19.dist-info/RECORD,,
+python3_commons-0.5.0.dist-info/AUTHORS.rst,sha256=3R9JnfjfjH5RoPWOeqKFJgxVShSSfzQPIrEr1nxIo9Q,90
+python3_commons-0.5.0.dist-info/LICENSE,sha256=xxILuojHm4fKQOrMHPSslbyy6WuKAN2RiG74HbrYfzM,34575
+python3_commons-0.5.0.dist-info/METADATA,sha256=xYXz86QMBSgEorZbSeUUiN2iICwrgItdxqWlX5de1Ik,974
+python3_commons-0.5.0.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
+python3_commons-0.5.0.dist-info/top_level.txt,sha256=lJI6sCBf68eUHzupCnn2dzG10lH3jJKTWM_hrN1cQ7M,16
+python3_commons-0.5.0.dist-info/RECORD,,

python3_commons/minio.py

@@ -1,28 +0,0 @@
-import urllib3
-from minio import Minio
-from pydantic import SecretStr
-
-
-def get_client(endpoint_url: str, region_name: str, access_key_id: SecretStr, secret_access_key: SecretStr,
-               secure: bool = True) -> Minio:
-    http_client = urllib3.PoolManager(
-        timeout=urllib3.util.Timeout(connect=300, read=300),
-        maxsize=50,
-        cert_reqs='CERT_REQUIRED' if secure else 'CERT_NONE',
-        retries=urllib3.Retry(
-            total=5,
-            backoff_factor=0.2,
-            status_forcelist=[500, 502, 503, 504]
-        )
-    )
-
-    client = Minio(
-        endpoint_url,
-        region=region_name,
-        access_key=access_key_id.get_secret_value(),
-        secret_key=secret_access_key.get_secret_value(),
-        http_client=http_client,
-        secure=secure
-    )
-
-    return client