python3-commons 0.6.7__py3-none-any.whl → 0.6.8__py3-none-any.whl

python3_commons/api_client.py

@@ -1,9 +1,9 @@
 from contextlib import asynccontextmanager
 from datetime import datetime, UTC
-from typing import Literal, Mapping
-from urllib.parse import urlencode
+from typing import AsyncGenerator, Literal, Mapping, Sequence
 
-import aiohttp
+from aiohttp import ClientSession
+from aiohttp.web_response import Response
 from pydantic import HttpUrl
 
 from python3_commons import audit
@@ -13,12 +13,16 @@ from python3_commons.helpers import request_to_curl
 
 @asynccontextmanager
 async def request(
-        base_url: HttpUrl,
-        uri: str,
-        params: Mapping | None = None,
-        method: Literal['get', 'post', 'patch', 'put', 'delete'] = 'get',
-        audit_name: str | None = None,
-):
+    client: ClientSession,
+    base_url: HttpUrl,
+    uri: str,
+    query: Mapping | None = None,
+    method: Literal['get', 'post', 'put', 'patch', 'options', 'head', 'delete'] = 'get',
+    headers: Mapping | None = None,
+    json: Mapping | Sequence | str | None = None,
+    data: bytes | None = None,
+    audit_name: str | None = None
+) -> AsyncGenerator[Response]:
     now = datetime.now(tz=UTC)
     date_path = now.strftime('%Y/%m/%d')
     timestamp = now.strftime('%H%M%S_%f')
@@ -26,19 +30,27 @@ async def request(
     uri_path = uri_path[1:] if uri_path.startswith('/') else uri_path
     url = f'{base_url}{uri}'
 
-    if params:
-        url_with_params = f'{url}?{urlencode(params)}'
+    if audit_name:
+        curl_request = None
 
-        if audit_name:
-            curl_request = request_to_curl(url_with_params, method, {}, None)
+        if method == 'get':
+            if query:
+                curl_request = request_to_curl(url, query, method, headers)
+        else:
+            curl_request = request_to_curl(url, query, method, headers, json, data)
+
+        if curl_request:
             await audit.write_audit_data(
                 s3_settings,
                 f'{date_path}/{audit_name}/{uri_path}/{method}_{timestamp}_request.txt',
                 curl_request.encode('utf-8')
             )
 
-    async with aiohttp.ClientSession() as client:
-        client_method = getattr(client, method)
+    client_method = getattr(client, method)
 
-        async with client_method(url, params=params) as response:
+    if method == 'get':
+        async with client_method(url, params=query) as response:
+            yield response
+    else:
+        async with client_method(url, params=query, json=json, data=data) as response:
             yield response

python3_commons/audit.py

@@ -3,6 +3,7 @@ import io
 import logging
 import tarfile
 from bz2 import BZ2Compressor
+from collections import deque
 from datetime import datetime, timedelta, UTC
 from typing import Generator, Iterable
 from uuid import uuid4
@@ -42,8 +43,7 @@ class GeneratedStream(io.BytesIO):
                 except StopIteration:
                     break
                 else:
-                    self.write(chunk)
-                    total_written_size += len(chunk)
+                    total_written_size += self.write(chunk)
 
         self.seek(0)
 
@@ -69,7 +69,7 @@ class GeneratedStream(io.BytesIO):
 
 def generate_archive(objects: Iterable[tuple[str, datetime, bytes]],
                      chunk_size: int = 4096) -> Generator[bytes, None, None]:
-    buffer = io.BytesIO()
+    buffer = deque()
 
     with tarfile.open(fileobj=buffer, mode='w') as archive:
         for name, last_modified, content in objects:
@@ -79,18 +79,22 @@ def generate_archive(objects: Iterable[tuple[str, datetime, bytes]],
             info.mtime = last_modified.timestamp()
             archive.addfile(info, io.BytesIO(content))
 
-            buffer.seek(0)
+            buffer_length = buffer.tell()
 
-            while True:
+            while buffer_length >= chunk_size:
+                buffer.seek(0)
                 chunk = buffer.read(chunk_size)
+                chunk_len = len(chunk)
 
                 if not chunk:
                     break
 
                 yield chunk
 
-            buffer.seek(0)
-            buffer.truncate(0)
+                buffer.seek(0)
+                buffer.truncate(chunk_len)
+                buffer.seek(0, io.SEEK_END)
+                buffer_length = buffer.tell()
 
     while True:
         chunk = buffer.read(chunk_size)
@@ -145,7 +149,7 @@ async def archive_audit_data(root_path: str = 'audit'):
     if objects := object_storage.get_objects(bucket_name, date_path, recursive=True):
         logger.info(f'Compacting files in: {date_path}')
 
-        generator = generate_archive(objects, chunk_size=5*1024*1024)
+        generator = generate_archive(objects, chunk_size=900_000)
         bzip2_generator = generate_bzip2(generator)
         archive_stream = GeneratedStream(bzip2_generator)
 

python3_commons/helpers.py

@@ -1,10 +1,13 @@
-import datetime
 import logging
 import shlex
 import threading
-
+from datetime import date, datetime, timedelta
 from decimal import Decimal, ROUND_HALF_UP
-from typing import Mapping
+from json import dumps
+from typing import Literal, Mapping, Sequence
+from urllib.parse import urlencode
+
+from python3_commons.serializers.json import CustomJSONEncoder
 
 logger = logging.getLogger(__name__)
 
@@ -30,23 +33,23 @@ class SingletonMeta(type):
                     return instance
 
 
-def date_from_string(string: str, fmt: str = '%d.%m.%Y') -> datetime.date:
+def date_from_string(string: str, fmt: str = '%d.%m.%Y') -> date:
     try:
-        return datetime.datetime.strptime(string, fmt).date()
+        return datetime.strptime(string, fmt).date()
     except ValueError:
-        return datetime.date.fromisoformat(string)
+        return date.fromisoformat(string)
 
 
-def datetime_from_string(string: str) -> datetime.datetime:
+def datetime_from_string(string: str) -> datetime:
     try:
-        return datetime.datetime.strptime(string, '%d.%m.%Y %H:%M:%S')
+        return datetime.strptime(string, '%d.%m.%Y %H:%M:%S')
     except ValueError:
-        return datetime.datetime.fromisoformat(string)
+        return datetime.fromisoformat(string)
 
 
 def date_range(start_date, end_date):
     for n in range(int((end_date - start_date).days + 1)):
-        yield start_date + datetime.timedelta(days=n)
+        yield start_date + timedelta(days=n)
 
 
 def tries(times):
@@ -72,16 +75,33 @@ def round_decimal(value: Decimal, decimal_places=2, rounding_mode=ROUND_HALF_UP)
         return value
 
 
-def request_to_curl(url: str, method: str, headers: Mapping, body: bytes | None = None) -> str:
-    curl_cmd = ['curl', '-i', '-X', method, shlex.quote(url)]
+def request_to_curl(
+        url: str,
+        query: Mapping | None = None,
+        method: Literal['get', 'post', 'put', 'patch', 'options', 'head', 'delete'] = 'get',
+        headers: Mapping | None = None,
+        json: Mapping | Sequence | str | None = None,
+        data: bytes | None = None
+) -> str:
+    if query:
+        url = f'{url}?{urlencode(query)}'
+
+    curl_cmd = ['curl', '-i', '-X', method.upper(), shlex.quote(url)]
+
+    if headers:
+        for key, value in headers.items():
+            header_line = f'{key}: {value}'
+            curl_cmd.append('-H')
+            curl_cmd.append(shlex.quote(header_line))
 
-    for key, value in headers.items():
-        header_line = f'{key}: {value}'
+    if json:
         curl_cmd.append('-H')
-        curl_cmd.append(shlex.quote(header_line))
+        curl_cmd.append(shlex.quote('Content-Type: application/json'))
 
-    if body is not None:
-        curl_cmd.append('--data')
-        curl_cmd.append(shlex.quote(body.decode('utf-8')))
+        curl_cmd.append('-d')
+        curl_cmd.append(shlex.quote(dumps(json, cls=CustomJSONEncoder)))
+    elif data:
+        curl_cmd.append('-d')
+        curl_cmd.append(shlex.quote(data.decode('utf-8')))
 
     return ' '.join(curl_cmd)

python3_commons/object_storage.py

@@ -42,7 +42,7 @@ def get_absolute_path(path: str) -> str:
     return path
 
 
-def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int, part_size: int = 0) -> str:
+def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int, part_size: int = 0) -> str | None:
     if s3_client := ObjectStorage(s3_settings).get_client():
         result = s3_client.put_object(bucket_name, path, data, length, part_size=part_size)
 

{python3_commons-0.6.7.dist-info → python3_commons-0.6.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: python3-commons
-Version: 0.6.7
+Version: 0.6.8
 Summary: Re-usable Python3 code
 Author-email: Oleg Korsak <kamikaze.is.waiting.you@gmail.com>
 License: gpl-3

{python3_commons-0.6.7.dist-info → python3_commons-0.6.8.dist-info}/RECORD

@@ -1,11 +1,11 @@
 python3_commons/__init__.py,sha256=0KgaYU46H_IMKn-BuasoRN3C4Hi45KlkHHoPbU9cwiA,189
-python3_commons/api_client.py,sha256=7DXXviUdinw80aTTn11Eh-goGmVdnAcaZsI27qVb2nQ,1420
-python3_commons/audit.py,sha256=cKQesMo01T_5OSxig-ZRyTXeCQjIjCepKeMgK8xtQ4o,6027
+python3_commons/api_client.py,sha256=0q6pViDNpYwL7zmaw-Fyil18y9QXYsCKJkoWCUM-uW0,1860
+python3_commons/audit.py,sha256=ZQcKkQcycQv5PT3tdKwVi43oxAxKBQwqvmavOUXoyHU,6231
 python3_commons/conf.py,sha256=qm2a2yWOhfawicBPjWnUett8TrsMtoyQXDxEJ_N-v-Y,637
 python3_commons/db.py,sha256=qhaDIdzBWgFyeP_XPKfHZlYVlwS2bpBPYMv84yV6820,738
 python3_commons/fs.py,sha256=wfLjybXndwLqNlOxTpm_HRJnuTcC4wbrHEOaEeCo9Wc,337
-python3_commons/helpers.py,sha256=hZG8M-mltBC8I9yx5ZuAM7bABFNuOsqX6FzSaQz4y9U,2480
-python3_commons/object_storage.py,sha256=pk2J14RL9FLTwaks-IS4EJX9TBMLzid35CroGftLNhU,4067
+python3_commons/helpers.py,sha256=OmuCF7UeJ6oe-rH1Y4ZYVW_uPQ973lCLsikj01wmNqs,3101
+python3_commons/object_storage.py,sha256=Bte49twIJ4l6VAzIqK6tLx7DFYwijErUmmhWkrZpSJE,4074
 python3_commons/logging/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 python3_commons/logging/filters.py,sha256=fuyjXZAUm-i2MNrxvFYag8F8Rr27x8W8MdV3ke6miSs,175
 python3_commons/logging/formatters.py,sha256=UXmmh1yd5Kc2dpvSHn6uCWLDWE2LMjlYAaH8cg3siV4,720
@@ -13,9 +13,9 @@ python3_commons/serializers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 python3_commons/serializers/json.py,sha256=P288wWz9ic38QWEMrpp_uwKPYkQiOgvE1cI4WZn6ZCg,808
 python3_commons/serializers/msgpack.py,sha256=tzIGGyDL3UpZnnouCtnxuYDx6InKM_C3PP1N4PN8wd4,1269
 python3_commons/serializers/msgspec.py,sha256=FuZVqOLJb0-lEKrs7dtjhwEbHIpfMUk5yu1hD64zRdc,2038
-python3_commons-0.6.7.dist-info/AUTHORS.rst,sha256=3R9JnfjfjH5RoPWOeqKFJgxVShSSfzQPIrEr1nxIo9Q,90
-python3_commons-0.6.7.dist-info/LICENSE,sha256=xxILuojHm4fKQOrMHPSslbyy6WuKAN2RiG74HbrYfzM,34575
-python3_commons-0.6.7.dist-info/METADATA,sha256=6L7NNsUmrmT5EOpjKf_dh605zkUZ8ltIMRn-lq0j5ME,985
-python3_commons-0.6.7.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-python3_commons-0.6.7.dist-info/top_level.txt,sha256=lJI6sCBf68eUHzupCnn2dzG10lH3jJKTWM_hrN1cQ7M,16
-python3_commons-0.6.7.dist-info/RECORD,,
+python3_commons-0.6.8.dist-info/AUTHORS.rst,sha256=3R9JnfjfjH5RoPWOeqKFJgxVShSSfzQPIrEr1nxIo9Q,90
+python3_commons-0.6.8.dist-info/LICENSE,sha256=xxILuojHm4fKQOrMHPSslbyy6WuKAN2RiG74HbrYfzM,34575
+python3_commons-0.6.8.dist-info/METADATA,sha256=8ZUT2mhKdXeUn7SUZhkm8pXZDL1O1-QToZMSCwMwWDU,985
+python3_commons-0.6.8.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+python3_commons-0.6.8.dist-info/top_level.txt,sha256=lJI6sCBf68eUHzupCnn2dzG10lH3jJKTWM_hrN1cQ7M,16
+python3_commons-0.6.8.dist-info/RECORD,,