tracktolib 0.51.1__tar.gz → 0.52.0__tar.gz

{tracktolib-0.51.1 → tracktolib-0.52.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tracktolib
-Version: 0.51.1
+Version: 0.52.0
 Summary: Utility library for python
 Home-page: https://github.com/tracktor/tracktolib
 License: MIT
@@ -12,6 +12,7 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Provides-Extra: api
 Provides-Extra: http
 Provides-Extra: logs

{tracktolib-0.51.1 → tracktolib-0.52.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tracktolib"
-version = "0.51.1"
+version = "0.52.0"
 description = "Utility library for python"
 authors = ["Julien Brayere <julien.brayere@tracktor.fr>"]
 license = "MIT"
@@ -56,6 +56,9 @@ psycopg = "^3.2.1"
 minio = "^7.2.7"
 pycryptodome = "^3.20.0"
 aiobotocore = "^2.13.1"
+deepdiff = "^8.0.1"
+python-json-logger = "^3.2.0"
+
 
 [build-system]
 requires = ["poetry-core"]
@@ -81,7 +84,7 @@ pythonPlatform = "Linux"
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "0.51.1"
+version = "0.52.0"
 tag_format = "$version"
 version_files = [
     "pyproject.toml:version"

{tracktolib-0.51.1 → tracktolib-0.52.0}/tracktolib/s3/s3.py

@@ -1,7 +1,7 @@
+import datetime as dt
 from io import BytesIO
 from pathlib import Path
-import datetime as dt
-from typing import TypedDict, Literal
+from typing import TypedDict, Literal, Callable
 
 try:
     from aiobotocore.client import AioBaseClient
@@ -36,18 +36,48 @@ async def upload_file(
     return resp
 
 
-async def download_file(client: AioBaseClient, bucket: str, path: str) -> BytesIO | None:
+type ContentLength = int
+type ChunkSize = int
+type OnUpdateDownload = Callable[[ChunkSize], None]
+type OnStartDownload = Callable[[ContentLength], None]
+
+
+async def download_file(
+    client: AioBaseClient,
+    bucket: str,
+    path: str,
+    *,
+    chunk_size: int = -1,
+    on_start: OnStartDownload | None = None,
+    on_update: OnUpdateDownload | None = None,
+) -> BytesIO | None:
     """
-    Loads a file from a s3 bucket
+    Loads a file from a s3 bucket.
+    If chunk_size is -1, the file will be loaded in one go
+    otherwise, the file will be loaded in chunks of size `chunk_size`.
+    When downloading in chunked, you can specify an `on_start` and `on_update`
+    callback to get the total size of the file and the size of each chunk downloaded respectively.
     """
     try:
         resp = await client.get_object(Bucket=bucket, Key=path)  # type: ignore
     except client.exceptions.NoSuchKey:
-        _file = None
-    else:
-        async with resp["Body"] as stream:
+        return None
+
+    if on_start is not None:
+        on_start(resp["ContentLength"])
+
+    async with resp["Body"] as stream:
+        if chunk_size == -1:
             _data = await stream.read()
             _file = BytesIO(_data)
+        else:
+            chunks = []
+            while chunk := await stream.content.read(chunk_size):
+                chunks.append(chunk)
+                if on_update is not None:
+                    on_update(len(chunk))
+            _file = BytesIO(b"".join(chunks)) if chunks else None
+
     return _file
 
 

{tracktolib-0.51.1 → tracktolib-0.52.0}/tracktolib/utils.py

@@ -10,20 +10,49 @@ import subprocess
 from decimal import Decimal
 from ipaddress import IPv4Address, IPv6Address
 from pathlib import Path
-from typing import Iterable, TypeVar, Iterator, Literal, overload, Any
+from typing import Iterable, TypeVar, Iterator, Literal, overload, Any, Callable
+
 
 T = TypeVar("T")
 
+type OnCmdUpdate = Callable[[str], None]
+type OnCmdDone = Callable[[str, str, int], None]
+
 
-def exec_cmd(cmd: str | list[str], *, encoding: str = "utf-8", env: dict | None = None) -> str:
+def exec_cmd(
+    cmd: str | list[str],
+    *,
+    env: dict | None = None,
+    on_update: OnCmdUpdate | None = None,
+    on_done: OnCmdDone | None = None,
+    **kwargs,
+) -> str:
     default_shell = os.getenv("SHELL", "/bin/bash")
 
-    stdout, stderr = subprocess.Popen(
-        cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, executable=default_shell, env=env
-    ).communicate()
-    if stderr:
-        raise Exception(stderr.decode(encoding))
-    return stdout.decode(encoding)
+    process = subprocess.Popen(
+        cmd,
+        shell=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        executable=default_shell,
+        env=env,
+        text=True,
+        **kwargs,
+    )
+
+    if on_update is not None:
+        for line in process.stderr or []:
+            on_update(line)
+
+    stdout, stderr = process.communicate()
+    exit_code = process.wait()
+
+    if on_done is not None:
+        on_done(stdout, stderr, exit_code)
+
+    if exit_code != 0:
+        raise Exception(stderr)
+    return stdout
 
 
 async def aexec_cmd(cmd: str | list[str], *, encoding: str = "utf-8", env: dict | None = None) -> str: