python3-commons 0.3.19__tar.gz → 0.4.0__tar.gz

{python3_commons-0.3.19/src/python3_commons.egg-info → python3_commons-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python3-commons
-Version: 0.3.19
+Version: 0.4.0
 Summary: Re-usable Python3 code
 Home-page: https://github.com/kamikaze/python3-commons
 Author: Oleg Korsak
@@ -18,7 +18,7 @@ Requires-Dist: asyncpg==0.29.0
 Requires-Dist: minio==7.2.7
 Requires-Dist: msgpack==1.0.8
 Requires-Dist: msgspec==0.18.6
-Requires-Dist: pydantic[email]==2.7.1
+Requires-Dist: pydantic[email]==2.7.2
 Requires-Dist: pydantic-settings==2.2.1
 Provides-Extra: testing
 Requires-Dist: pytest; extra == "testing"

{python3_commons-0.3.19 → python3_commons-0.4.0}/requirements.txt

@@ -2,5 +2,5 @@ asyncpg==0.29.0
 minio==7.2.7
 msgpack==1.0.8
 msgspec==0.18.6
-pydantic[email]==2.7.1
+pydantic[email]==2.7.2
 pydantic-settings==2.2.1

{python3_commons-0.3.19 → python3_commons-0.4.0}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = python3-commons
-version = 0.3.19
+version = 0.4.0
 description = Re-usable Python3 code
 author = Oleg Korsak
 author_email = kamikaze.is.waiting.you@gmail.com
@@ -27,7 +27,7 @@ install_requires =
 	minio==7.2.7
 	msgpack==1.0.8
 	msgspec==0.18.6
-	pydantic[email]==2.7.1
+	pydantic[email]==2.7.2
 	pydantic-settings==2.2.1
 python_requires = >=3.12
 

python3_commons-0.4.0/src/python3_commons/audit.py

@@ -0,0 +1,44 @@
+import logging
+import tarfile
+from datetime import date, timedelta
+from io import BytesIO
+
+from python3_commons import object_storage
+from python3_commons.conf import s3_settings
+
+logger = logging.getLogger(__name__)
+
+
+async def archive_audit_data(root_path: str = 'audit'):
+    today = date.today() - timedelta(days=1)
+    year = today.year
+    month = today.month
+    day = today.day
+    bucket_name = s3_settings.s3_bucket
+    fo = BytesIO()
+    object_names = []
+    date_path = object_storage.get_absolute_path(f'{root_path}/{year}/{month:02}/{day:02}')
+
+    with tarfile.open(fileobj=fo, mode='w|bz2') as archive:
+        if objects := object_storage.get_objects(bucket_name, date_path, recursive=True):
+            logger.info(f'Compacting files in: {date_path}')
+
+            for name, last_modified, content in objects:
+                info = tarfile.TarInfo(name)
+                info.size = len(content)
+                info.mtime = last_modified.timestamp()
+                archive.addfile(info, BytesIO(content))
+                object_names.append(name)
+
+    fo.seek(0)
+
+    if object_names:
+        archive_path = object_storage.get_absolute_path(
+            f'.archive/{year}_{month:02}/{year}_{month:02}_{day:02}.tar.bz2')
+        object_storage.put_object(bucket_name, archive_path, fo, fo.getbuffer().nbytes)
+
+        if errors := object_storage.remove_objects(bucket_name, object_names=object_names):
+            for error in errors:
+                logger.error(f'Failed to delete object in {bucket_name=}: {error}')
+    else:
+        logger.info('No objects to archive found.')

{python3_commons-0.3.19 → python3_commons-0.4.0}/src/python3_commons/conf.py

@@ -15,6 +15,7 @@ class S3Settings(BaseSettings):
     s3_secure: bool = True
     s3_bucket: str | None = None
     s3_bucket_root: str | None = None
+    s3_cert_verify: bool = True
 
 
 settings = CommonSettings()

{python3_commons-0.3.19 → python3_commons-0.4.0}/src/python3_commons/object_storage.py

@@ -1,29 +1,29 @@
+import io
 import logging
 from datetime import datetime
-from io import IOBase
-from typing import Generator, Iterable, Any
+from typing import Generator, Iterable
 
-from minio import Minio
+from minio import Minio, S3Error
 from minio.datatypes import Object
-from minio.deleteobjects import DeleteObject
+from minio.deleteobjects import DeleteObject, DeleteError
 
-from python3_commons import minio
-from python3_commons.conf import s3_settings
+from python3_commons.conf import s3_settings, S3Settings
 
 logger = logging.getLogger(__name__)
 __CLIENT = None
 
 
-def get_s3_client() -> Minio:
+def get_s3_client(settings: S3Settings) -> Minio:
     global __CLIENT
 
     if not __CLIENT and s3_settings.s3_endpoint_url:
-        __CLIENT = minio.get_client(
-            s3_settings.s3_endpoint_url,
-            s3_settings.s3_region_name,
-            s3_settings.s3_access_key_id,
-            s3_settings.s3_secret_access_key,
-            s3_settings.s3_secure
+        __CLIENT = Minio(
+            settings.s3_endpoint_url,
+            region=settings.s3_region_name,
+            access_key=settings.s3_access_key_id.get_secret_value(),
+            secret_key=settings.s3_secret_access_key.get_secret_value(),
+            secure=settings.s3_secure,
+            cert_check=settings.s3_cert_verify
         )
 
     return __CLIENT
@@ -34,13 +34,13 @@ def get_absolute_path(path: str) -> str:
         path = path[1:]
 
     if bucket_root := s3_settings.s3_bucket_root:
-        path = f'{bucket_root[:1] if bucket_root.startswith("/") else bucket_root}/{path}'
+        path = f'{bucket_root[:1] if bucket_root.startswith('/') else bucket_root}/{path}'
 
     return path
 
 
-def put_object(bucket_name: str, path: str, data: IOBase, length: int) -> str:
-    s3_client = get_s3_client()
+def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int) -> str:
+    s3_client = get_s3_client(s3_settings)
 
     if s3_client:
         result = s3_client.put_object(bucket_name, path, data, length)
@@ -53,7 +53,7 @@ def put_object(bucket_name: str, path: str, data: IOBase, length: int) -> str:
 
 
 def get_object_stream(bucket_name: str, path: str):
-    s3_client = get_s3_client()
+    s3_client = get_s3_client(s3_settings)
 
     if s3_client:
         logger.debug(f'Getting object from object storage: {bucket_name}:{path}')
@@ -85,7 +85,7 @@ def get_object(bucket_name: str, path: str) -> bytes:
 
 
 def list_objects(bucket_name: str, prefix: str, recursive: bool = True) -> Generator[Object, None, None]:
-    s3_client = get_s3_client()
+    s3_client = get_s3_client(s3_settings)
 
     yield from s3_client.list_objects(bucket_name, prefix=prefix, recursive=recursive)
 
@@ -104,13 +104,13 @@ def get_objects(bucket_name: str, path: str,
 
 
 def remove_object(bucket_name: str, object_name: str):
-    s3_client = get_s3_client()
+    s3_client = get_s3_client(s3_settings)
     s3_client.remove_object(bucket_name, object_name)
 
 
 def remove_objects(bucket_name: str, prefix: str = None,
-                   object_names: Iterable[str] = None) -> Generator[Any, Any, None] | None:
-    s3_client = get_s3_client()
+                   object_names: Iterable[str] = None) -> Iterable[DeleteError] | None:
+    s3_client = get_s3_client(s3_settings)
 
     if prefix:
         delete_object_list = map(
@@ -125,3 +125,17 @@ def remove_objects(bucket_name: str, prefix: str = None,
     errors = s3_client.remove_objects(bucket_name, delete_object_list)
 
     return errors
+
+
+async def store_bytes_in_s3(settings: S3Settings, data: bytes, key: str):
+    if settings.s3_secret_access_key:
+        try:
+            client = get_s3_client(settings)
+
+            client.put_object(settings.s3_bucket, key, io.BytesIO(data), len(data))
+        except S3Error as e:
+            logger.error(f'Failed storing object in storage: {e}')
+        else:
+            logger.debug(f'Stored object in storage: {key}')
+    else:
+        logger.debug(f'S3 is not configured, not storing object in storage: {key}')

{python3_commons-0.3.19 → python3_commons-0.4.0/src/python3_commons.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python3-commons
-Version: 0.3.19
+Version: 0.4.0
 Summary: Re-usable Python3 code
 Home-page: https://github.com/kamikaze/python3-commons
 Author: Oleg Korsak
@@ -18,7 +18,7 @@ Requires-Dist: asyncpg==0.29.0
 Requires-Dist: minio==7.2.7
 Requires-Dist: msgpack==1.0.8
 Requires-Dist: msgspec==0.18.6
-Requires-Dist: pydantic[email]==2.7.1
+Requires-Dist: pydantic[email]==2.7.2
 Requires-Dist: pydantic-settings==2.2.1
 Provides-Extra: testing
 Requires-Dist: pytest; extra == "testing"

{python3_commons-0.3.19 → python3_commons-0.4.0}/src/python3_commons.egg-info/SOURCES.txt

@@ -20,11 +20,11 @@ docs/index.rst
 docs/license.rst
 docs/_static/.gitignore
 src/python3_commons/__init__.py
+src/python3_commons/audit.py
 src/python3_commons/conf.py
 src/python3_commons/db.py
 src/python3_commons/fs.py
 src/python3_commons/helpers.py
-src/python3_commons/minio.py
 src/python3_commons/object_storage.py
 src/python3_commons.egg-info/PKG-INFO
 src/python3_commons.egg-info/SOURCES.txt

{python3_commons-0.3.19 → python3_commons-0.4.0}/src/python3_commons.egg-info/requires.txt

@@ -2,7 +2,7 @@ asyncpg==0.29.0
 minio==7.2.7
 msgpack==1.0.8
 msgspec==0.18.6
-pydantic[email]==2.7.1
+pydantic[email]==2.7.2
 pydantic-settings==2.2.1
 
 [testing]

python3_commons-0.3.19/src/python3_commons/minio.py

@@ -1,28 +0,0 @@
-import urllib3
-from minio import Minio
-from pydantic import SecretStr
-
-
-def get_client(endpoint_url: str, region_name: str, access_key_id: SecretStr, secret_access_key: SecretStr,
-               secure: bool = True) -> Minio:
-    http_client = urllib3.PoolManager(
-        timeout=urllib3.util.Timeout(connect=300, read=300),
-        maxsize=50,
-        cert_reqs='CERT_REQUIRED' if secure else 'CERT_NONE',
-        retries=urllib3.Retry(
-            total=5,
-            backoff_factor=0.2,
-            status_forcelist=[500, 502, 503, 504]
-        )
-    )
-
-    client = Minio(
-        endpoint_url,
-        region=region_name,
-        access_key=access_key_id.get_secret_value(),
-        secret_key=secret_access_key.get_secret_value(),
-        http_client=http_client,
-        secure=secure
-    )
-
-    return client