tracktolib 0.51.0__py3-none-any.whl → 0.52.0__py3-none-any.whl

tracktolib/pg/query.py

@@ -206,10 +206,6 @@ def get_update_fields(
     _merge_keys = set(merge_keys or [])
     _ignore_keys = ignore_keys or []
 
-    _invalid_merge_keys = _merge_keys - set(keys)
-    if _invalid_merge_keys:
-        raise ValueError(f"Merge keys not in keys found: {_invalid_merge_keys}")
-
     for k in keys:
         v = item[k]
         if k in _ignore_keys:
@@ -397,8 +393,6 @@ async def update_one(
     query = PGUpdateQuery(
         table=table, items=[item], start_from=start_from, where_keys=keys, where=where, merge_keys=merge_keys
     )
-    print(query.query)
-    print(query.values)
     await conn.execute(query.query, *args, *query.values)
 
 

tracktolib/s3/s3.py

@@ -1,7 +1,7 @@
+import datetime as dt
 from io import BytesIO
 from pathlib import Path
-import datetime as dt
-from typing import TypedDict, Literal
+from typing import TypedDict, Literal, Callable
 
 try:
     from aiobotocore.client import AioBaseClient
@@ -36,18 +36,48 @@ async def upload_file(
     return resp
 
 
-async def download_file(client: AioBaseClient, bucket: str, path: str) -> BytesIO | None:
+type ContentLength = int
+type ChunkSize = int
+type OnUpdateDownload = Callable[[ChunkSize], None]
+type OnStartDownload = Callable[[ContentLength], None]
+
+
+async def download_file(
+    client: AioBaseClient,
+    bucket: str,
+    path: str,
+    *,
+    chunk_size: int = -1,
+    on_start: OnStartDownload | None = None,
+    on_update: OnUpdateDownload | None = None,
+) -> BytesIO | None:
     """
-    Loads a file from a s3 bucket
+    Loads a file from a s3 bucket.
+    If chunk_size is -1, the file will be loaded in one go
+    otherwise, the file will be loaded in chunks of size `chunk_size`.
+    When downloading in chunked, you can specify an `on_start` and `on_update`
+    callback to get the total size of the file and the size of each chunk downloaded respectively.
     """
     try:
         resp = await client.get_object(Bucket=bucket, Key=path)  # type: ignore
     except client.exceptions.NoSuchKey:
-        _file = None
-    else:
-        async with resp["Body"] as stream:
+        return None
+
+    if on_start is not None:
+        on_start(resp["ContentLength"])
+
+    async with resp["Body"] as stream:
+        if chunk_size == -1:
             _data = await stream.read()
             _file = BytesIO(_data)
+        else:
+            chunks = []
+            while chunk := await stream.content.read(chunk_size):
+                chunks.append(chunk)
+                if on_update is not None:
+                    on_update(len(chunk))
+            _file = BytesIO(b"".join(chunks)) if chunks else None
+
     return _file
 
 

tracktolib/utils.py

@@ -10,20 +10,49 @@ import subprocess
 from decimal import Decimal
 from ipaddress import IPv4Address, IPv6Address
 from pathlib import Path
-from typing import Iterable, TypeVar, Iterator, Literal, overload, Any
+from typing import Iterable, TypeVar, Iterator, Literal, overload, Any, Callable
+
 
 T = TypeVar("T")
 
+type OnCmdUpdate = Callable[[str], None]
+type OnCmdDone = Callable[[str, str, int], None]
+
 
-def exec_cmd(cmd: str | list[str], *, encoding: str = "utf-8", env: dict | None = None) -> str:
+def exec_cmd(
+    cmd: str | list[str],
+    *,
+    env: dict | None = None,
+    on_update: OnCmdUpdate | None = None,
+    on_done: OnCmdDone | None = None,
+    **kwargs,
+) -> str:
     default_shell = os.getenv("SHELL", "/bin/bash")
 
-    stdout, stderr = subprocess.Popen(
-        cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, executable=default_shell, env=env
-    ).communicate()
-    if stderr:
-        raise Exception(stderr.decode(encoding))
-    return stdout.decode(encoding)
+    process = subprocess.Popen(
+        cmd,
+        shell=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        executable=default_shell,
+        env=env,
+        text=True,
+        **kwargs,
+    )
+
+    if on_update is not None:
+        for line in process.stderr or []:
+            on_update(line)
+
+    stdout, stderr = process.communicate()
+    exit_code = process.wait()
+
+    if on_done is not None:
+        on_done(stdout, stderr, exit_code)
+
+    if exit_code != 0:
+        raise Exception(stderr)
+    return stdout
 
 
 async def aexec_cmd(cmd: str | list[str], *, encoding: str = "utf-8", env: dict | None = None) -> str:

{tracktolib-0.51.0.dist-info → tracktolib-0.52.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tracktolib
-Version: 0.51.0
+Version: 0.52.0
 Summary: Utility library for python
 Home-page: https://github.com/tracktor/tracktolib
 License: MIT
@@ -12,6 +12,7 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Provides-Extra: api
 Provides-Extra: http
 Provides-Extra: logs

{tracktolib-0.51.0.dist-info → tracktolib-0.52.0.dist-info}/RECORD

@@ -4,16 +4,16 @@ tracktolib/api.py,sha256=xArVgRj_g7bw2tEDbjC9qs9f55b9X0-kJoc8-s1rsYo,9616
 tracktolib/http_utils.py,sha256=c10JGmHaBw3VSDMYhz2dvVw2lo4PUAq1xMub74I7xDc,2625
 tracktolib/logs.py,sha256=M5RZ8OYKgLEBJeC1AwUCSRbFAH09hwdTjjs-AQ9QGg8,2204
 tracktolib/pg/__init__.py,sha256=j67e3B3gBbCHLD20QBybptmNdbbVMzNhZE6XjIPuKVo,349
-tracktolib/pg/query.py,sha256=XX5eD02TRWRIiBVj0ENooQ893aycan2fJ55_1J207Rw,14711
+tracktolib/pg/query.py,sha256=MBYOTL9aLnYRBhaLy8-P6uGNXOmyluejNyUdt3Gfx5M,14504
 tracktolib/pg/utils.py,sha256=cL24KEt4SWJQ7LJPzaO3c8Xg0ZLmjhn22DtTWg86nwc,6324
 tracktolib/pg_sync.py,sha256=z6EB0hELq6t129gnVnX0rNXv2McZXxI8khy-TEG65Ag,5165
 tracktolib/pg_utils.py,sha256=VXPpy1jGq6aCgTlfFJDIrq6JDujR83JN5ZRiCi8Lx4E,2582
 tracktolib/s3/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tracktolib/s3/minio.py,sha256=wMEjkSes9Fp39fD17IctALpD6zB2xwDRQEmO7Vzan3g,1387
-tracktolib/s3/s3.py,sha256=d0Q63Zb62ef4jAt05zQwpgYvAoxHC9kSQuHzzK90VvE,4825
+tracktolib/s3/s3.py,sha256=I2fT3C9pIm9pM0V96O_wbYg_bPJX35ryzx05G3EgZus,5816
 tracktolib/tests.py,sha256=Pbc4yGQrIWFLBEgX-kDBxhMCqH-bfOJtkH2agvaM6ZQ,382
-tracktolib/utils.py,sha256=jwLww8bqDu8zEip9uN4yW0lE5_YMWfrAHYtagr8sYOA,5295
-tracktolib-0.51.0.dist-info/LICENSE,sha256=uUanH0X7SeZEPdsRTHegMSMTiIHMurt9H0jSwEwKE1Y,1081
-tracktolib-0.51.0.dist-info/METADATA,sha256=v2MsoN7EIuclo58jhPp40YJespn9Oy4rYKM8kg7VRsQ,3641
-tracktolib-0.51.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-tracktolib-0.51.0.dist-info/RECORD,,
+tracktolib/utils.py,sha256=ysTBF9V35fVXQVBPk0kfE_84SGRxzrayqmg9RbtoJq4,5761
+tracktolib-0.52.0.dist-info/LICENSE,sha256=uUanH0X7SeZEPdsRTHegMSMTiIHMurt9H0jSwEwKE1Y,1081
+tracktolib-0.52.0.dist-info/METADATA,sha256=peqROd6d1t-a0mXJFJjgqy0mc7trrpLb8-l4nYANf5s,3692
+tracktolib-0.52.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+tracktolib-0.52.0.dist-info/RECORD,,

{tracktolib-0.51.0.dist-info → tracktolib-0.52.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.0
+Generator: poetry-core 1.9.1
 Root-Is-Purelib: true
 Tag: py3-none-any