python3-commons 0.6.6__tar.gz → 0.6.8__tar.gz

{python3_commons-0.6.6/src/python3_commons.egg-info → python3_commons-0.6.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: python3-commons
-Version: 0.6.6
+Version: 0.6.8
 Summary: Re-usable Python3 code
 Author-email: Oleg Korsak <kamikaze.is.waiting.you@gmail.com>
 License: gpl-3

{python3_commons-0.6.6 → python3_commons-0.6.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "python3-commons"
-version = "0.6.6"
+version = "0.6.8"
 description = "Re-usable Python3 code"
 authors = [
     {name = "Oleg Korsak", email = "kamikaze.is.waiting.you@gmail.com"}
@@ -48,7 +48,6 @@ exclude = ["tests"]
 
 [tool.pytest.ini_options]
 addopts = [
-    "--cov python3_commons --cov-report term-missing",
     "--verbose"
 ]
 norecursedirs = ["dist", "build", ".tox"]

python3_commons-0.6.8/src/python3_commons/api_client.py

@@ -0,0 +1,56 @@
+from contextlib import asynccontextmanager
+from datetime import datetime, UTC
+from typing import AsyncGenerator, Literal, Mapping, Sequence
+
+from aiohttp import ClientSession
+from aiohttp.web_response import Response
+from pydantic import HttpUrl
+
+from python3_commons import audit
+from python3_commons.conf import s3_settings
+from python3_commons.helpers import request_to_curl
+
+
+@asynccontextmanager
+async def request(
+    client: ClientSession,
+    base_url: HttpUrl,
+    uri: str,
+    query: Mapping | None = None,
+    method: Literal['get', 'post', 'put', 'patch', 'options', 'head', 'delete'] = 'get',
+    headers: Mapping | None = None,
+    json: Mapping | Sequence | str | None = None,
+    data: bytes | None = None,
+    audit_name: str | None = None
+) -> AsyncGenerator[Response]:
+    now = datetime.now(tz=UTC)
+    date_path = now.strftime('%Y/%m/%d')
+    timestamp = now.strftime('%H%M%S_%f')
+    uri_path = uri[:-1] if uri.endswith('/') else uri
+    uri_path = uri_path[1:] if uri_path.startswith('/') else uri_path
+    url = f'{base_url}{uri}'
+
+    if audit_name:
+        curl_request = None
+
+        if method == 'get':
+            if query:
+                curl_request = request_to_curl(url, query, method, headers)
+        else:
+            curl_request = request_to_curl(url, query, method, headers, json, data)
+
+        if curl_request:
+            await audit.write_audit_data(
+                s3_settings,
+                f'{date_path}/{audit_name}/{uri_path}/{method}_{timestamp}_request.txt',
+                curl_request.encode('utf-8')
+            )
+
+    client_method = getattr(client, method)
+
+    if method == 'get':
+        async with client_method(url, params=query) as response:
+            yield response
+    else:
+        async with client_method(url, params=query, json=json, data=data) as response:
+            yield response

{python3_commons-0.6.6 → python3_commons-0.6.8}/src/python3_commons/audit.py

@@ -3,6 +3,7 @@ import io
 import logging
 import tarfile
 from bz2 import BZ2Compressor
+from collections import deque
 from datetime import datetime, timedelta, UTC
 from typing import Generator, Iterable
 from uuid import uuid4
@@ -42,8 +43,7 @@ class GeneratedStream(io.BytesIO):
                 except StopIteration:
                     break
                 else:
-                    self.write(chunk)
-                    total_written_size += len(chunk)
+                    total_written_size += self.write(chunk)
 
         self.seek(0)
 
@@ -69,7 +69,7 @@ class GeneratedStream(io.BytesIO):
 
 def generate_archive(objects: Iterable[tuple[str, datetime, bytes]],
                      chunk_size: int = 4096) -> Generator[bytes, None, None]:
-    buffer = io.BytesIO()
+    buffer = deque()
 
     with tarfile.open(fileobj=buffer, mode='w') as archive:
         for name, last_modified, content in objects:
@@ -79,18 +79,22 @@ def generate_archive(objects: Iterable[tuple[str, datetime, bytes]],
             info.mtime = last_modified.timestamp()
             archive.addfile(info, io.BytesIO(content))
 
-            buffer.seek(0)
+            buffer_length = buffer.tell()
 
-            while True:
+            while buffer_length >= chunk_size:
+                buffer.seek(0)
                 chunk = buffer.read(chunk_size)
+                chunk_len = len(chunk)
 
                 if not chunk:
                     break
 
                 yield chunk
 
-            buffer.seek(0)
-            buffer.truncate(0)
+                buffer.seek(0)
+                buffer.truncate(chunk_len)
+                buffer.seek(0, io.SEEK_END)
+                buffer_length = buffer.tell()
 
     while True:
         chunk = buffer.read(chunk_size)
@@ -145,7 +149,7 @@ async def archive_audit_data(root_path: str = 'audit'):
     if objects := object_storage.get_objects(bucket_name, date_path, recursive=True):
         logger.info(f'Compacting files in: {date_path}')
 
-        generator = generate_archive(objects, chunk_size=5*1024*1024)
+        generator = generate_archive(objects, chunk_size=900_000)
         bzip2_generator = generate_bzip2(generator)
         archive_stream = GeneratedStream(bzip2_generator)
 

{python3_commons-0.6.6 → python3_commons-0.6.8}/src/python3_commons/helpers.py

@@ -1,10 +1,13 @@
-import datetime
 import logging
 import shlex
 import threading
-
+from datetime import date, datetime, timedelta
 from decimal import Decimal, ROUND_HALF_UP
-from typing import Mapping
+from json import dumps
+from typing import Literal, Mapping, Sequence
+from urllib.parse import urlencode
+
+from python3_commons.serializers.json import CustomJSONEncoder
 
 logger = logging.getLogger(__name__)
 
@@ -30,23 +33,23 @@ class SingletonMeta(type):
                     return instance
 
 
-def date_from_string(string: str, fmt: str = '%d.%m.%Y') -> datetime.date:
+def date_from_string(string: str, fmt: str = '%d.%m.%Y') -> date:
     try:
-        return datetime.datetime.strptime(string, fmt).date()
+        return datetime.strptime(string, fmt).date()
     except ValueError:
-        return datetime.date.fromisoformat(string)
+        return date.fromisoformat(string)
 
 
-def datetime_from_string(string: str) -> datetime.datetime:
+def datetime_from_string(string: str) -> datetime:
     try:
-        return datetime.datetime.strptime(string, '%d.%m.%Y %H:%M:%S')
+        return datetime.strptime(string, '%d.%m.%Y %H:%M:%S')
     except ValueError:
-        return datetime.datetime.fromisoformat(string)
+        return datetime.fromisoformat(string)
 
 
 def date_range(start_date, end_date):
     for n in range(int((end_date - start_date).days + 1)):
-        yield start_date + datetime.timedelta(days=n)
+        yield start_date + timedelta(days=n)
 
 
 def tries(times):
@@ -72,16 +75,33 @@ def round_decimal(value: Decimal, decimal_places=2, rounding_mode=ROUND_HALF_UP)
         return value
 
 
-def request_to_curl(url: str, method: str, headers: Mapping, body: bytes | None = None) -> str:
-    curl_cmd = ['curl', '-i', '-X', method, shlex.quote(url)]
+def request_to_curl(
+        url: str,
+        query: Mapping | None = None,
+        method: Literal['get', 'post', 'put', 'patch', 'options', 'head', 'delete'] = 'get',
+        headers: Mapping | None = None,
+        json: Mapping | Sequence | str | None = None,
+        data: bytes | None = None
+) -> str:
+    if query:
+        url = f'{url}?{urlencode(query)}'
+
+    curl_cmd = ['curl', '-i', '-X', method.upper(), shlex.quote(url)]
+
+    if headers:
+        for key, value in headers.items():
+            header_line = f'{key}: {value}'
+            curl_cmd.append('-H')
+            curl_cmd.append(shlex.quote(header_line))
 
-    for key, value in headers.items():
-        header_line = f'{key}: {value}'
+    if json:
         curl_cmd.append('-H')
-        curl_cmd.append(shlex.quote(header_line))
+        curl_cmd.append(shlex.quote('Content-Type: application/json'))
 
-    if body is not None:
-        curl_cmd.append('--data')
-        curl_cmd.append(shlex.quote(body.decode('utf-8')))
+        curl_cmd.append('-d')
+        curl_cmd.append(shlex.quote(dumps(json, cls=CustomJSONEncoder)))
+    elif data:
+        curl_cmd.append('-d')
+        curl_cmd.append(shlex.quote(data.decode('utf-8')))
 
     return ' '.join(curl_cmd)

{python3_commons-0.6.6 → python3_commons-0.6.8}/src/python3_commons/object_storage.py

@@ -42,7 +42,7 @@ def get_absolute_path(path: str) -> str:
     return path
 
 
-def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int, part_size: int = 0) -> str:
+def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int, part_size: int = 0) -> str | None:
     if s3_client := ObjectStorage(s3_settings).get_client():
         result = s3_client.put_object(bucket_name, path, data, length, part_size=part_size)
 

{python3_commons-0.6.6 → python3_commons-0.6.8}/src/python3_commons/serializers/msgspec.py

@@ -7,7 +7,7 @@ from datetime import datetime, date
 from typing import Any
 
 from msgspec import msgpack
-from msgspec.msgpack import Ext
+from msgspec.msgpack import encode, Ext
 
 from python3_commons.serializers.json import CustomJSONEncoder
 
@@ -42,6 +42,20 @@ def ext_hook(code: int, data: memoryview) -> Any:
 
 MSGPACK_ENCODER = msgpack.Encoder(enc_hook=enc_hook)
 MSGPACK_DECODER = msgpack.Decoder(ext_hook=ext_hook)
+MSGPACK_DECODER_NATIVE = msgpack.Decoder()
+
+
+def serialize_msgpack_native(data) -> bytes:
+    return encode(data)
+
+
+def deserialize_msgpack_native(data: bytes, data_type=None):
+    if data_type:
+        result = msgpack.decode(data, type=data_type)
+    else:
+        result = MSGPACK_DECODER_NATIVE.decode(data)
+
+    return result
 
 
 def serialize_msgpack(data) -> bytes:

{python3_commons-0.6.6 → python3_commons-0.6.8/src/python3_commons.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: python3-commons
-Version: 0.6.6
+Version: 0.6.8
 Summary: Re-usable Python3 code
 Author-email: Oleg Korsak <kamikaze.is.waiting.you@gmail.com>
 License: gpl-3

{python3_commons-0.6.6 → python3_commons-0.6.8}/tests/conftest.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 from datetime import datetime, date
 from decimal import Decimal
+from uuid import UUID, uuid4
 
 import msgspec
 import pytest
@@ -40,6 +41,11 @@ def data_dataclass():
     )
 
 
+class SubStruc(msgspec.Struct):
+    uid: UUID
+    name: str
+
+
 class TestStruct(msgspec.Struct):
     a: int
     b: str
@@ -47,17 +53,19 @@ class TestStruct(msgspec.Struct):
     d: datetime
     e: date
     f: Decimal
+    sub: SubStruc
 
 
 @pytest.fixture
-def data_struct():
+def data_struct() -> TestStruct:
     return TestStruct(
         a=1,
         b='B',
         c=None,
         d=datetime(2023, 7, 25, 1, 2, 3),
         e=date(2023, 7, 24),
-        f=Decimal('1.23')
+        f=Decimal('1.23'),
+        sub=SubStruc(uid=uuid4(), name='sub-struct')
     )
 
 

python3_commons-0.6.8/tests/test_msgspec.py

@@ -0,0 +1,56 @@
+from decimal import Decimal
+
+from python3_commons.serializers import msgspec
+
+
+def test_encode_decode_dict_to_msgpack(data_dict):
+    """
+    enc_hook is not being called on complex types like dict
+    """
+    expected_result = {
+        'A': 1,
+        'B': 'B',
+        'C': None,
+        'D': '2023-07-25T01:02:03',
+        'E': '2023-07-24',
+        'F': '1.23',
+    }
+    binary_data = msgspec.serialize_msgpack(data_dict)
+
+    assert msgspec.deserialize_msgpack(binary_data) == data_dict
+
+
+def test_encode_decode_dataclass_to_msgpack(data_dataclass):
+    binary_data = msgspec.serialize_msgpack(data_dataclass)
+
+    assert msgspec.deserialize_msgpack(binary_data, data_type=data_dataclass.__class__) == data_dataclass
+
+
+def test_encode_decode_struct_to_msgpack(data_struct):
+    binary_data = msgspec.serialize_msgpack(data_struct)
+    decoded_struct = msgspec.deserialize_msgpack(binary_data, data_struct.__class__)
+
+    assert decoded_struct == data_struct
+
+
+def test_encode_decode_struct_to_msgpack_native(data_struct):
+    binary_data = msgspec.serialize_msgpack_native(data_struct)
+    decoded_struct = msgspec.deserialize_msgpack_native(binary_data, data_struct.__class__)
+
+    assert decoded_struct == data_struct
+
+
+def test_encode_decode_decimal_to_msgpack():
+    value = Decimal('1.2345')
+    binary_data = msgspec.serialize_msgpack(value)
+    decoded_value = msgspec.deserialize_msgpack(binary_data, Decimal)
+
+    assert decoded_value == value
+
+
+def test_encode_decode_str_to_msgpack():
+    value = '1.2345'
+    binary_data = msgspec.serialize_msgpack(value)
+    decoded_value = msgspec.deserialize_msgpack(binary_data)
+
+    assert decoded_value == value

python3_commons-0.6.6/src/python3_commons/api_client.py

@@ -1,44 +0,0 @@
-from contextlib import asynccontextmanager
-from datetime import datetime, UTC
-from typing import Literal, Mapping
-from urllib.parse import urlencode
-
-import aiohttp
-from pydantic import HttpUrl
-
-from python3_commons import audit
-from python3_commons.conf import s3_settings
-from python3_commons.helpers import request_to_curl
-
-
-@asynccontextmanager
-async def request(
-        base_url: HttpUrl,
-        uri: str,
-        params: Mapping | None = None,
-        method: Literal['get', 'post', 'patch', 'put', 'delete'] = 'get',
-        audit_name: str | None = None,
-):
-    now = datetime.now(tz=UTC)
-    date_path = now.strftime('%Y/%m/%d')
-    timestamp = now.strftime('%H%M%S_%f')
-    uri_path = uri[:-1] if uri.endswith('/') else uri
-    uri_path = uri_path[1:] if uri_path.startswith('/') else uri_path
-    url = f'{base_url}{uri}'
-
-    if params:
-        url_with_params = f'{url}?{urlencode(params)}'
-
-        if audit_name:
-            curl_request = request_to_curl(url_with_params, method, {}, None)
-            await audit.write_audit_data(
-                s3_settings,
-                f'{date_path}/{audit_name}/{uri_path}/{method}_{timestamp}_request.txt',
-                curl_request.encode('utf-8')
-            )
-
-    async with aiohttp.ClientSession() as client:
-        client_method = getattr(client, method)
-
-        async with client_method(url, params=params) as response:
-            yield response

python3_commons-0.6.6/tests/test_msgspec.py

@@ -1,33 +0,0 @@
-from msgspec import Struct
-
-from python3_commons.serializers import msgspec
-
-
-class AStruct(Struct):
-    pass
-
-
-def test_encode_decode_dict_to_msgpack(data_dict):
-    expected_result = {
-        'A': 1,
-        'B': 'B',
-        'C': None,
-        'D': '2023-07-25T01:02:03',
-        'E': '2023-07-24',
-        'F': '1.23',
-    }
-    binary_data = msgspec.serialize_msgpack(data_dict)
-
-    assert msgspec.deserialize_msgpack(binary_data) == expected_result
-
-
-def test_encode_decode_dataclass_to_msgpack(data_dataclass):
-    binary_data = msgspec.serialize_msgpack(data_dataclass)
-
-    assert msgspec.deserialize_msgpack(binary_data, data_type=data_dataclass.__class__) == data_dataclass
-
-
-def test_encode_decode_struct_to_msgpack(data_struct):
-    binary_data = msgspec.serialize_msgpack(data_struct)
-
-    assert msgspec.deserialize_msgpack(binary_data, data_type=data_struct.__class__) == data_struct