tracktolib 0.51.1__tar.gz → 0.53.1__tar.gz

{tracktolib-0.51.1 → tracktolib-0.53.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tracktolib
-Version: 0.51.1
+Version: 0.53.1
 Summary: Utility library for python
 Home-page: https://github.com/tracktor/tracktolib
 License: MIT
@@ -12,6 +12,7 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Provides-Extra: api
 Provides-Extra: http
 Provides-Extra: logs
@@ -22,14 +23,14 @@ Provides-Extra: s3-minio
 Provides-Extra: tests
 Requires-Dist: aiobotocore (>=2.9.0) ; extra == "s3"
 Requires-Dist: asyncpg (>=0.27.0) ; extra == "pg"
-Requires-Dist: deepdiff (>=6.6.0) ; extra == "tests"
+Requires-Dist: deepdiff (>=8.1.0) ; extra == "tests"
 Requires-Dist: fastapi (>=0.103.2) ; extra == "api"
 Requires-Dist: httpx (>=0.25.0) ; extra == "http"
 Requires-Dist: minio (>=7.2.0) ; extra == "s3-minio"
 Requires-Dist: psycopg (>=3.1.12) ; extra == "pg-sync"
 Requires-Dist: pycryptodome (>=3.20.0) ; extra == "s3-minio"
 Requires-Dist: pydantic (>=2) ; extra == "api"
-Requires-Dist: python-json-logger (>=2.0.4) ; extra == "logs"
+Requires-Dist: python-json-logger (>=3.2.1) ; extra == "logs"
 Requires-Dist: rich (>=13.6.0) ; extra == "pg"
 Project-URL: Repository, https://github.com/tracktor/tracktolib
 Description-Content-Type: text/markdown

{tracktolib-0.51.1 → tracktolib-0.53.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tracktolib"
-version = "0.51.1"
+version = "0.53.1"
 description = "Utility library for python"
 authors = ["Julien Brayere <julien.brayere@tracktor.fr>"]
 license = "MIT"
@@ -18,13 +18,13 @@ include = [
 [tool.poetry.dependencies]
 python = ">=3.12,<4.0"
 
-python-json-logger = { version = ">=2.0.4", optional = true }
+python-json-logger = { version = ">=3.2.1", optional = true }
 psycopg = { version = ">=3.1.12", optional = true }
 minio = { version = ">=7.2.0", optional = true }
 # Needed for minio, otherwise it raises ModuleNotFoundError: No module named 'Crypto.Cipher'
 pycryptodome =  { version = ">=3.20.0", optional = true }
 aiobotocore = { version = ">=2.9.0", optional = true }
-deepdiff = { version = ">=6.6.0", optional = true }
+deepdiff = { version = ">=8.1.0", optional = true }
 httpx = { version = ">=0.25.0", optional = true }
 fastapi = { version = ">=0.103.2", optional = true }
 pydantic = { version = ">=2", optional = true }
@@ -56,6 +56,8 @@ psycopg = "^3.2.1"
 minio = "^7.2.7"
 pycryptodome = "^3.20.0"
 aiobotocore = "^2.13.1"
+python-json-logger = "^3.2.0"
+
 
 [build-system]
 requires = ["poetry-core"]
@@ -81,7 +83,7 @@ pythonPlatform = "Linux"
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "0.51.1"
+version = "0.53.1"
 tag_format = "$version"
 version_files = [
     "pyproject.toml:version"

{tracktolib-0.51.1 → tracktolib-0.53.1}/tracktolib/logs.py

@@ -3,7 +3,7 @@ from typing import Literal, overload, Any, TypeGuard
 from dataclasses import dataclass
 
 try:
-    from pythonjsonlogger import jsonlogger
+    from pythonjsonlogger.json import JsonFormatter
 except ImportError:
     raise ImportError('Please install pythonjsonlogger or tracktolib with "log" to use this module')
 
@@ -11,7 +11,7 @@ LogFormat = Literal["json", "console"]
 
 
 @dataclass
-class CustomJsonFormatter(jsonlogger.JsonFormatter):
+class CustomJsonFormatter(JsonFormatter):
     version: str
 
     def __init__(self, version: str, *args, **kwargs):

{tracktolib-0.51.1 → tracktolib-0.53.1}/tracktolib/s3/s3.py

@@ -1,7 +1,7 @@
+import datetime as dt
 from io import BytesIO
 from pathlib import Path
-import datetime as dt
-from typing import TypedDict, Literal
+from typing import TypedDict, Literal, Callable
 
 try:
     from aiobotocore.client import AioBaseClient
@@ -36,18 +36,48 @@ async def upload_file(
     return resp
 
 
-async def download_file(client: AioBaseClient, bucket: str, path: str) -> BytesIO | None:
+type ContentLength = int
+type ChunkSize = int
+type OnUpdateDownload = Callable[[ChunkSize], None]
+type OnStartDownload = Callable[[ContentLength], None]
+
+
+async def download_file(
+    client: AioBaseClient,
+    bucket: str,
+    path: str,
+    *,
+    chunk_size: int = -1,
+    on_start: OnStartDownload | None = None,
+    on_update: OnUpdateDownload | None = None,
+) -> BytesIO | None:
     """
-    Loads a file from a s3 bucket
+    Loads a file from a s3 bucket.
+    If chunk_size is -1, the file will be loaded in one go
+    otherwise, the file will be loaded in chunks of size `chunk_size`.
+    When downloading in chunked, you can specify an `on_start` and `on_update`
+    callback to get the total size of the file and the size of each chunk downloaded respectively.
     """
     try:
         resp = await client.get_object(Bucket=bucket, Key=path)  # type: ignore
     except client.exceptions.NoSuchKey:
-        _file = None
-    else:
-        async with resp["Body"] as stream:
+        return None
+
+    if on_start is not None:
+        on_start(resp["ContentLength"])
+
+    async with resp["Body"] as stream:
+        if chunk_size == -1:
             _data = await stream.read()
             _file = BytesIO(_data)
+        else:
+            chunks = []
+            while chunk := await stream.content.read(chunk_size):
+                chunks.append(chunk)
+                if on_update is not None:
+                    on_update(len(chunk))
+            _file = BytesIO(b"".join(chunks)) if chunks else None
+
     return _file
 
 
@@ -120,6 +150,7 @@ async def list_files(
     search_query: str | None = None,
     max_items: int | None = None,
     page_size: int | None = None,
+    starting_token: str | None = None,
 ) -> list[S3Item]:
     """
     See https://jmespath.org/ for the search query syntax.
@@ -128,9 +159,11 @@ async def list_files(
     paginator = client.get_paginator("list_objects")
     config = {}
     if max_items is not None:
-        config["max_items"] = max_items
+        config["MaxItems"] = max_items
     if page_size is not None:
-        config["page_size"] = page_size
+        config["PageSize"] = page_size
+    if starting_token is not None:
+        config["StartingToken"] = starting_token
 
     page_iterator = paginator.paginate(Bucket=bucket, Prefix=path, PaginationConfig=config if config else {})
     filtered_iterator = page_iterator.search(search_query) if search_query else page_iterator

{tracktolib-0.51.1 → tracktolib-0.53.1}/tracktolib/tests.py

@@ -2,7 +2,7 @@ import pprint
 from typing import Iterable
 
 try:
-    import deepdiff
+    import deepdiff.diff as deepdiff
 except ImportError:
     raise ImportError('Please install deepdiff or tracktolib with "tests" to use this module')
 

{tracktolib-0.51.1 → tracktolib-0.53.1}/tracktolib/utils.py

@@ -10,20 +10,49 @@ import subprocess
 from decimal import Decimal
 from ipaddress import IPv4Address, IPv6Address
 from pathlib import Path
-from typing import Iterable, TypeVar, Iterator, Literal, overload, Any
+from typing import Iterable, TypeVar, Iterator, Literal, overload, Any, Callable
+
 
 T = TypeVar("T")
 
+type OnCmdUpdate = Callable[[str], None]
+type OnCmdDone = Callable[[str, str, int], None]
+
 
-def exec_cmd(cmd: str | list[str], *, encoding: str = "utf-8", env: dict | None = None) -> str:
+def exec_cmd(
+    cmd: str | list[str],
+    *,
+    env: dict | None = None,
+    on_update: OnCmdUpdate | None = None,
+    on_done: OnCmdDone | None = None,
+    **kwargs,
+) -> str:
     default_shell = os.getenv("SHELL", "/bin/bash")
 
-    stdout, stderr = subprocess.Popen(
-        cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, executable=default_shell, env=env
-    ).communicate()
-    if stderr:
-        raise Exception(stderr.decode(encoding))
-    return stdout.decode(encoding)
+    process = subprocess.Popen(
+        cmd,
+        shell=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        executable=default_shell,
+        env=env,
+        text=True,
+        **kwargs,
+    )
+
+    if on_update is not None:
+        for line in process.stderr or []:
+            on_update(line)
+
+    stdout, stderr = process.communicate()
+    exit_code = process.wait()
+
+    if on_done is not None:
+        on_done(stdout, stderr, exit_code)
+
+    if exit_code != 0:
+        raise Exception(stderr)
+    return stdout
 
 
 async def aexec_cmd(cmd: str | list[str], *, encoding: str = "utf-8", env: dict | None = None) -> str: