tracktolib 0.67.0__py3-none-any.whl → 0.69.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tracktolib/api.py +21 -22
- tracktolib/cf/__init__.py +8 -0
- tracktolib/cf/client.py +149 -0
- tracktolib/cf/types.py +17 -0
- tracktolib/gh/__init__.py +11 -0
- tracktolib/gh/client.py +206 -0
- tracktolib/gh/types.py +203 -0
- tracktolib/http_utils.py +1 -1
- tracktolib/logs.py +1 -1
- tracktolib/notion/__init__.py +44 -0
- tracktolib/notion/blocks.py +459 -0
- tracktolib/notion/cache.py +202 -0
- tracktolib/notion/fetch.py +121 -5
- tracktolib/notion/markdown.py +468 -0
- tracktolib/notion/models.py +26 -0
- tracktolib/notion/utils.py +567 -0
- tracktolib/pg/__init__.py +10 -10
- tracktolib/pg/query.py +1 -1
- tracktolib/pg/utils.py +5 -5
- tracktolib/pg_sync.py +5 -7
- tracktolib/pg_utils.py +1 -4
- tracktolib/s3/minio.py +1 -1
- tracktolib/s3/niquests.py +235 -32
- tracktolib/s3/s3.py +1 -1
- tracktolib/utils.py +48 -3
- {tracktolib-0.67.0.dist-info → tracktolib-0.69.0.dist-info}/METADATA +115 -2
- tracktolib-0.69.0.dist-info/RECORD +31 -0
- {tracktolib-0.67.0.dist-info → tracktolib-0.69.0.dist-info}/WHEEL +1 -1
- tracktolib-0.67.0.dist-info/RECORD +0 -21
tracktolib/s3/niquests.py
CHANGED
|
@@ -1,18 +1,21 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
from pathlib import Path
|
|
5
|
-
|
|
3
|
+
import hashlib
|
|
6
4
|
import http
|
|
7
5
|
import xml.etree.ElementTree as ET
|
|
8
6
|
from contextlib import asynccontextmanager
|
|
9
7
|
from dataclasses import dataclass, field
|
|
10
|
-
from
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import AsyncIterator, Awaitable, Callable, Literal, NamedTuple, Required, Self, TypedDict, Unpack
|
|
10
|
+
from urllib.parse import unquote
|
|
11
|
+
from xml.sax.saxutils import escape as xml_escape
|
|
11
12
|
|
|
12
13
|
try:
|
|
13
14
|
import botocore.client
|
|
14
15
|
import botocore.session
|
|
15
16
|
import jmespath
|
|
17
|
+
from botocore.auth import SigV4Auth
|
|
18
|
+
from botocore.awsrequest import AWSRequest
|
|
16
19
|
from botocore.config import Config
|
|
17
20
|
except ImportError as e:
|
|
18
21
|
raise ImportError("botocore is required for S3 operations. Install with tracktolib[s3-niquests]") from e
|
|
@@ -22,26 +25,37 @@ try:
|
|
|
22
25
|
except ImportError as e:
|
|
23
26
|
raise ImportError("niquests is required for S3 operations. Install with tracktolib[s3-niquests]") from e
|
|
24
27
|
|
|
28
|
+
try:
|
|
29
|
+
import ujson as json
|
|
30
|
+
except ImportError:
|
|
31
|
+
import json
|
|
32
|
+
|
|
25
33
|
from ..utils import get_stream_chunk
|
|
26
34
|
|
|
27
35
|
__all__ = (
|
|
28
|
-
"S3Session",
|
|
29
|
-
"s3_delete_object",
|
|
30
|
-
"s3_delete_objects",
|
|
31
|
-
"s3_list_files",
|
|
32
|
-
"s3_put_object",
|
|
33
|
-
"s3_get_object",
|
|
34
|
-
"s3_download_file",
|
|
35
|
-
"s3_upload_file",
|
|
36
|
-
"s3_create_multipart_upload",
|
|
37
|
-
"s3_multipart_upload",
|
|
38
|
-
"s3_file_upload",
|
|
39
36
|
"S3MultipartUpload",
|
|
40
37
|
"S3Object",
|
|
41
38
|
"S3ObjectParams",
|
|
39
|
+
"S3Session",
|
|
42
40
|
"UploadPart",
|
|
43
41
|
"build_s3_headers",
|
|
44
42
|
"build_s3_presigned_params",
|
|
43
|
+
"s3_create_multipart_upload",
|
|
44
|
+
"s3_delete_bucket_policy",
|
|
45
|
+
"s3_delete_bucket_website",
|
|
46
|
+
"s3_delete_object",
|
|
47
|
+
"s3_delete_objects",
|
|
48
|
+
"s3_download_file",
|
|
49
|
+
"s3_empty_bucket",
|
|
50
|
+
"s3_file_upload",
|
|
51
|
+
"s3_get_bucket_policy",
|
|
52
|
+
"s3_get_object",
|
|
53
|
+
"s3_list_files",
|
|
54
|
+
"s3_multipart_upload",
|
|
55
|
+
"s3_put_bucket_policy",
|
|
56
|
+
"s3_put_bucket_website",
|
|
57
|
+
"s3_put_object",
|
|
58
|
+
"s3_upload_file",
|
|
45
59
|
)
|
|
46
60
|
|
|
47
61
|
ACL = Literal[
|
|
@@ -193,16 +207,16 @@ class S3Session:
|
|
|
193
207
|
s3_config: Config | None = None
|
|
194
208
|
s3_client: botocore.client.BaseClient | None = None
|
|
195
209
|
http_client: niquests.AsyncSession = field(default_factory=niquests.AsyncSession)
|
|
210
|
+
_botocore_session: botocore.session.Session | None = field(default=None, init=False, repr=False)
|
|
196
211
|
|
|
197
212
|
def __post_init__(self):
|
|
198
213
|
if self.s3_client is None:
|
|
199
|
-
|
|
200
|
-
self.
|
|
214
|
+
self._botocore_session = botocore.session.Session()
|
|
215
|
+
self._botocore_session.set_credentials(self.access_key, self.secret_key)
|
|
216
|
+
self.s3_client = self._botocore_session.create_client(
|
|
201
217
|
"s3",
|
|
202
218
|
endpoint_url=self.endpoint_url,
|
|
203
219
|
region_name=self.region,
|
|
204
|
-
aws_access_key_id=self.access_key,
|
|
205
|
-
aws_secret_access_key=self.secret_key,
|
|
206
220
|
config=self.s3_config,
|
|
207
221
|
)
|
|
208
222
|
|
|
@@ -220,13 +234,13 @@ class S3Session:
|
|
|
220
234
|
await self.http_client.__aexit__(exc_type, exc_val, exc_tb)
|
|
221
235
|
self._s3.close()
|
|
222
236
|
|
|
223
|
-
async def delete_object(self, bucket: str, key: str) -> niquests.
|
|
237
|
+
async def delete_object(self, bucket: str, key: str) -> niquests.AsyncResponse:
|
|
224
238
|
"""Delete an object from S3."""
|
|
225
|
-
return await s3_delete_object(self._s3, self.http_client, bucket, key)
|
|
239
|
+
return await s3_delete_object(self._s3, self.http_client, bucket, key) # pyright: ignore[reportReturnType]
|
|
226
240
|
|
|
227
|
-
async def delete_objects(self, bucket: str, keys: list[str]) -> list[niquests.
|
|
241
|
+
async def delete_objects(self, bucket: str, keys: list[str]) -> list[niquests.AsyncResponse]:
|
|
228
242
|
"""Delete multiple objects from S3."""
|
|
229
|
-
return await s3_delete_objects(self._s3, self.http_client, bucket, keys)
|
|
243
|
+
return await s3_delete_objects(self._s3, self.http_client, bucket, keys) # pyright: ignore[reportReturnType]
|
|
230
244
|
|
|
231
245
|
def list_files(
|
|
232
246
|
self,
|
|
@@ -252,15 +266,15 @@ class S3Session:
|
|
|
252
266
|
|
|
253
267
|
async def put_object(
|
|
254
268
|
self, bucket: str, key: str, data: bytes, **kwargs: Unpack[S3ObjectParams]
|
|
255
|
-
) -> niquests.
|
|
269
|
+
) -> niquests.AsyncResponse:
|
|
256
270
|
"""Upload an object to S3."""
|
|
257
|
-
return await s3_put_object(self._s3, self.http_client, bucket, key, data, **kwargs)
|
|
271
|
+
return await s3_put_object(self._s3, self.http_client, bucket, key, data, **kwargs) # pyright: ignore[reportReturnType]
|
|
258
272
|
|
|
259
273
|
async def upload_file(
|
|
260
274
|
self, bucket: str, file: Path, path: str, **kwargs: Unpack[S3ObjectParams]
|
|
261
|
-
) -> niquests.
|
|
275
|
+
) -> niquests.AsyncResponse:
|
|
262
276
|
"""Upload a file to S3."""
|
|
263
|
-
return await s3_upload_file(self._s3, self.http_client, bucket, file, path, **kwargs)
|
|
277
|
+
return await s3_upload_file(self._s3, self.http_client, bucket, file, path, **kwargs) # pyright: ignore[reportReturnType]
|
|
264
278
|
|
|
265
279
|
async def get_object(self, bucket: str, key: str) -> bytes | None:
|
|
266
280
|
"""Download an object from S3."""
|
|
@@ -310,10 +324,33 @@ class S3Session:
|
|
|
310
324
|
**kwargs,
|
|
311
325
|
)
|
|
312
326
|
|
|
327
|
+
async def put_bucket_policy(self, bucket: str, policy: str | dict) -> niquests.AsyncResponse:
|
|
328
|
+
"""Set a bucket policy."""
|
|
329
|
+
return await s3_put_bucket_policy(self._s3, self.http_client, bucket, policy, self._botocore_session) # pyright: ignore[reportReturnType]
|
|
313
330
|
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
)
|
|
331
|
+
async def get_bucket_policy(self, bucket: str) -> dict | None:
|
|
332
|
+
"""Get a bucket policy. Returns None if no policy exists."""
|
|
333
|
+
return await s3_get_bucket_policy(self._s3, self.http_client, bucket)
|
|
334
|
+
|
|
335
|
+
async def delete_bucket_policy(self, bucket: str) -> niquests.AsyncResponse:
|
|
336
|
+
"""Delete a bucket policy."""
|
|
337
|
+
return await s3_delete_bucket_policy(self._s3, self.http_client, bucket) # pyright: ignore[reportReturnType]
|
|
338
|
+
|
|
339
|
+
async def put_bucket_website(
|
|
340
|
+
self, bucket: str, index_document: str = "index.html", error_document: str | None = None
|
|
341
|
+
) -> niquests.AsyncResponse:
|
|
342
|
+
"""Configure a bucket as a static website."""
|
|
343
|
+
return await s3_put_bucket_website(
|
|
344
|
+
self._s3, self.http_client, bucket, index_document, error_document, self._botocore_session
|
|
345
|
+
) # pyright: ignore[reportReturnType]
|
|
346
|
+
|
|
347
|
+
async def delete_bucket_website(self, bucket: str) -> niquests.AsyncResponse:
|
|
348
|
+
"""Remove website configuration from a bucket."""
|
|
349
|
+
return await s3_delete_bucket_website(self._s3, self.http_client, bucket) # pyright: ignore[reportReturnType]
|
|
350
|
+
|
|
351
|
+
async def empty_bucket(self, bucket: str, *, on_progress: Callable[[str], None] | None = None) -> int:
|
|
352
|
+
"""Delete all objects from a bucket. Returns count of deleted objects."""
|
|
353
|
+
return await s3_empty_bucket(self._s3, self.http_client, bucket, on_progress=on_progress)
|
|
317
354
|
|
|
318
355
|
|
|
319
356
|
class UploadPart(TypedDict):
|
|
@@ -321,8 +358,16 @@ class UploadPart(TypedDict):
|
|
|
321
358
|
ETag: str | None
|
|
322
359
|
|
|
323
360
|
|
|
361
|
+
class S3MultipartUpload(NamedTuple):
|
|
362
|
+
fetch_create: Callable[[], Awaitable[str]]
|
|
363
|
+
fetch_complete: Callable[[], Awaitable[niquests.Response]]
|
|
364
|
+
upload_part: Callable[[bytes], Awaitable[UploadPart]]
|
|
365
|
+
generate_presigned_url: Callable[..., str]
|
|
366
|
+
fetch_abort: Callable[[], Awaitable[niquests.Response]]
|
|
367
|
+
|
|
368
|
+
|
|
324
369
|
class S3Object(TypedDict, total=False):
|
|
325
|
-
Key: str
|
|
370
|
+
Key: Required[str]
|
|
326
371
|
LastModified: str
|
|
327
372
|
ETag: str
|
|
328
373
|
Size: int
|
|
@@ -398,12 +443,15 @@ async def s3_list_files(
|
|
|
398
443
|
|
|
399
444
|
page_items: list[S3Object] = []
|
|
400
445
|
for contents in root.findall("s3:Contents", ns):
|
|
401
|
-
item: S3Object = {}
|
|
446
|
+
item: S3Object = {} # pyright: ignore[reportAssignmentType]
|
|
402
447
|
for child in contents:
|
|
403
448
|
tag = child.tag.replace(f"{{{ns['s3']}}}", "")
|
|
404
449
|
item[tag] = child.text
|
|
405
450
|
if "Size" in item:
|
|
406
451
|
item["Size"] = int(item["Size"])
|
|
452
|
+
# URL-decode the Key (some S3-compatible servers like Garage return URL-encoded keys)
|
|
453
|
+
if "Key" in item and item["Key"]:
|
|
454
|
+
item["Key"] = unquote(item["Key"])
|
|
407
455
|
page_items.append(item)
|
|
408
456
|
|
|
409
457
|
if search_query:
|
|
@@ -676,3 +724,158 @@ async def s3_file_upload(
|
|
|
676
724
|
return
|
|
677
725
|
await mpart.upload_part(chunk)
|
|
678
726
|
has_uploaded_parts = True
|
|
727
|
+
|
|
728
|
+
|
|
729
|
+
def _get_credentials(s3: botocore.client.BaseClient, session: botocore.session.Session | None = None):
|
|
730
|
+
"""Get credentials from session (public API) or fall back to client internals."""
|
|
731
|
+
if session is not None:
|
|
732
|
+
return session.get_credentials()
|
|
733
|
+
# Fall back to private API for backward compatibility with standalone function usage
|
|
734
|
+
return s3._request_signer._credentials # pyright: ignore[reportAttributeAccessIssue]
|
|
735
|
+
|
|
736
|
+
|
|
737
|
+
def _sign_s3_request(
|
|
738
|
+
s3: botocore.client.BaseClient,
|
|
739
|
+
method: str,
|
|
740
|
+
url: str,
|
|
741
|
+
data: bytes,
|
|
742
|
+
content_type: str,
|
|
743
|
+
botocore_session: botocore.session.Session | None = None,
|
|
744
|
+
) -> dict[str, str]:
|
|
745
|
+
"""Create and sign an S3 request, returning the headers to use."""
|
|
746
|
+
request = AWSRequest(method=method, url=url, data=data, headers={"Content-Type": content_type})
|
|
747
|
+
request.headers["x-amz-content-sha256"] = hashlib.sha256(data).hexdigest()
|
|
748
|
+
|
|
749
|
+
credentials = _get_credentials(s3, botocore_session)
|
|
750
|
+
region = s3.meta.region_name
|
|
751
|
+
signer = SigV4Auth(credentials, "s3", region)
|
|
752
|
+
signer.add_auth(request)
|
|
753
|
+
|
|
754
|
+
return dict(request.headers)
|
|
755
|
+
|
|
756
|
+
|
|
757
|
+
async def s3_put_bucket_policy(
|
|
758
|
+
s3: botocore.client.BaseClient,
|
|
759
|
+
client: niquests.AsyncSession,
|
|
760
|
+
bucket: str,
|
|
761
|
+
policy: str | dict,
|
|
762
|
+
botocore_session: botocore.session.Session | None = None,
|
|
763
|
+
) -> niquests.Response:
|
|
764
|
+
"""
|
|
765
|
+
Set a bucket policy using a signed request.
|
|
766
|
+
|
|
767
|
+
The policy can be provided as a JSON string or a dict (which will be serialized).
|
|
768
|
+
If botocore_session is provided, credentials are retrieved via the public API;
|
|
769
|
+
otherwise falls back to the client's internal credentials.
|
|
770
|
+
"""
|
|
771
|
+
policy_str = policy if isinstance(policy, str) else json.dumps(policy)
|
|
772
|
+
policy_bytes = policy_str.encode("utf-8")
|
|
773
|
+
url = f"{s3.meta.endpoint_url}/{bucket}?policy"
|
|
774
|
+
headers = _sign_s3_request(s3, "PUT", url, policy_bytes, "application/json", botocore_session)
|
|
775
|
+
return (await client.put(url, data=policy_bytes, headers=headers)).raise_for_status()
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
async def s3_get_bucket_policy(
|
|
779
|
+
s3: botocore.client.BaseClient,
|
|
780
|
+
client: niquests.AsyncSession,
|
|
781
|
+
bucket: str,
|
|
782
|
+
) -> dict | None:
|
|
783
|
+
"""
|
|
784
|
+
Get a bucket policy using presigned URL.
|
|
785
|
+
|
|
786
|
+
Returns the policy as a dict, or None if no policy exists.
|
|
787
|
+
"""
|
|
788
|
+
url = s3.generate_presigned_url(
|
|
789
|
+
ClientMethod="get_bucket_policy",
|
|
790
|
+
Params={"Bucket": bucket},
|
|
791
|
+
)
|
|
792
|
+
resp = await client.get(url)
|
|
793
|
+
if resp.status_code == http.HTTPStatus.NOT_FOUND:
|
|
794
|
+
return None
|
|
795
|
+
# NoSuchBucketPolicy returns 404 on AWS, but some providers may return other codes
|
|
796
|
+
if resp.status_code == http.HTTPStatus.NO_CONTENT:
|
|
797
|
+
return None
|
|
798
|
+
resp.raise_for_status()
|
|
799
|
+
if resp.content is None:
|
|
800
|
+
return None
|
|
801
|
+
return json.loads(resp.content)
|
|
802
|
+
|
|
803
|
+
|
|
804
|
+
async def s3_delete_bucket_policy(
|
|
805
|
+
s3: botocore.client.BaseClient,
|
|
806
|
+
client: niquests.AsyncSession,
|
|
807
|
+
bucket: str,
|
|
808
|
+
) -> niquests.Response:
|
|
809
|
+
"""Delete a bucket policy using presigned URL."""
|
|
810
|
+
url = s3.generate_presigned_url(
|
|
811
|
+
ClientMethod="delete_bucket_policy",
|
|
812
|
+
Params={"Bucket": bucket},
|
|
813
|
+
)
|
|
814
|
+
return (await client.delete(url)).raise_for_status()
|
|
815
|
+
|
|
816
|
+
|
|
817
|
+
def _build_website_xml(index_document: str, error_document: str | None, api_version: str) -> str:
|
|
818
|
+
"""Build XML payload for S3 website configuration."""
|
|
819
|
+
index_xml = f"<IndexDocument><Suffix>{xml_escape(index_document)}</Suffix></IndexDocument>"
|
|
820
|
+
error_xml = f"<ErrorDocument><Key>{xml_escape(error_document)}</Key></ErrorDocument>" if error_document else ""
|
|
821
|
+
return f'<WebsiteConfiguration xmlns="http://s3.amazonaws.com/doc/{api_version}/">{index_xml}{error_xml}</WebsiteConfiguration>'
|
|
822
|
+
|
|
823
|
+
|
|
824
|
+
async def s3_put_bucket_website(
|
|
825
|
+
s3: botocore.client.BaseClient,
|
|
826
|
+
client: niquests.AsyncSession,
|
|
827
|
+
bucket: str,
|
|
828
|
+
index_document: str = "index.html",
|
|
829
|
+
error_document: str | None = None,
|
|
830
|
+
botocore_session: botocore.session.Session | None = None,
|
|
831
|
+
) -> niquests.AsyncResponse:
|
|
832
|
+
"""
|
|
833
|
+
Configure a bucket as a static website using a signed request.
|
|
834
|
+
|
|
835
|
+
Note: This operation is not supported by MinIO.
|
|
836
|
+
If botocore_session is provided, credentials are retrieved via the public API;
|
|
837
|
+
otherwise falls back to the client's internal credentials.
|
|
838
|
+
"""
|
|
839
|
+
api_version = s3.meta.service_model.api_version
|
|
840
|
+
xml_payload = _build_website_xml(index_document, error_document, api_version)
|
|
841
|
+
xml_bytes = xml_payload.encode("utf-8")
|
|
842
|
+
url = f"{s3.meta.endpoint_url}/{bucket}?website"
|
|
843
|
+
headers = _sign_s3_request(s3, "PUT", url, xml_bytes, "application/xml", botocore_session)
|
|
844
|
+
return (await client.put(url, data=xml_bytes, headers=headers)).raise_for_status() # pyright: ignore[reportReturnType]
|
|
845
|
+
|
|
846
|
+
|
|
847
|
+
async def s3_delete_bucket_website(
|
|
848
|
+
s3: botocore.client.BaseClient,
|
|
849
|
+
client: niquests.AsyncSession,
|
|
850
|
+
bucket: str,
|
|
851
|
+
) -> niquests.Response:
|
|
852
|
+
"""Remove website configuration from a bucket using presigned URL."""
|
|
853
|
+
url = s3.generate_presigned_url(
|
|
854
|
+
ClientMethod="delete_bucket_website",
|
|
855
|
+
Params={"Bucket": bucket},
|
|
856
|
+
)
|
|
857
|
+
return (await client.delete(url)).raise_for_status()
|
|
858
|
+
|
|
859
|
+
|
|
860
|
+
async def s3_empty_bucket(
|
|
861
|
+
s3: botocore.client.BaseClient,
|
|
862
|
+
client: niquests.AsyncSession,
|
|
863
|
+
bucket: str,
|
|
864
|
+
*,
|
|
865
|
+
on_progress: Callable[[str], None] | None = None,
|
|
866
|
+
) -> int:
|
|
867
|
+
"""
|
|
868
|
+
Delete all objects from a bucket.
|
|
869
|
+
|
|
870
|
+
The optional on_progress callback is called with each deleted key.
|
|
871
|
+
Returns the count of deleted objects.
|
|
872
|
+
"""
|
|
873
|
+
deleted_count = 0
|
|
874
|
+
async for obj in s3_list_files(s3, client, bucket, ""):
|
|
875
|
+
key = obj.get("Key")
|
|
876
|
+
if key:
|
|
877
|
+
await s3_delete_object(s3, client, bucket, key)
|
|
878
|
+
deleted_count += 1
|
|
879
|
+
if on_progress:
|
|
880
|
+
on_progress(key)
|
|
881
|
+
return deleted_count
|
tracktolib/s3/s3.py
CHANGED
tracktolib/utils.py
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
from pathlib import Path
|
|
2
|
-
|
|
3
1
|
import asyncio
|
|
4
2
|
import datetime as dt
|
|
5
3
|
import importlib.util
|
|
@@ -10,8 +8,19 @@ import subprocess
|
|
|
10
8
|
import sys
|
|
11
9
|
from decimal import Decimal
|
|
12
10
|
from ipaddress import IPv4Address, IPv6Address
|
|
11
|
+
from pathlib import Path
|
|
13
12
|
from types import ModuleType
|
|
14
|
-
from typing import
|
|
13
|
+
from typing import (
|
|
14
|
+
Any,
|
|
15
|
+
AsyncIterable,
|
|
16
|
+
AsyncIterator,
|
|
17
|
+
Callable,
|
|
18
|
+
Coroutine,
|
|
19
|
+
Iterable,
|
|
20
|
+
Iterator,
|
|
21
|
+
Literal,
|
|
22
|
+
overload,
|
|
23
|
+
)
|
|
15
24
|
|
|
16
25
|
type OnCmdUpdate = Callable[[str], None]
|
|
17
26
|
type OnCmdDone = Callable[[str, str, int], None]
|
|
@@ -231,3 +240,39 @@ def deep_reload(m: ModuleType):
|
|
|
231
240
|
def get_first_line(lines: str) -> str:
|
|
232
241
|
_lines = lines.split("\n")
|
|
233
242
|
return _lines[0] if _lines else lines
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
async def run_coros[R](
|
|
246
|
+
coros: Iterable[Coroutine[Any, Any, R]],
|
|
247
|
+
sem: asyncio.Semaphore | None = None,
|
|
248
|
+
) -> AsyncIterator[R]:
|
|
249
|
+
"""Run coroutines and yield results in order.
|
|
250
|
+
|
|
251
|
+
Args:
|
|
252
|
+
coros: Coroutines to execute
|
|
253
|
+
sem: If provided, run in parallel with rate limiting.
|
|
254
|
+
If None, run sequentially.
|
|
255
|
+
|
|
256
|
+
Yields:
|
|
257
|
+
Results in input order.
|
|
258
|
+
|
|
259
|
+
Example:
|
|
260
|
+
async for result in run_coros([fetch(1), fetch(2)], sem):
|
|
261
|
+
print(result)
|
|
262
|
+
"""
|
|
263
|
+
coro_list = list(coros)
|
|
264
|
+
|
|
265
|
+
if sem is None:
|
|
266
|
+
for coro in coro_list:
|
|
267
|
+
yield await coro
|
|
268
|
+
else:
|
|
269
|
+
|
|
270
|
+
async def with_sem(coro: Coroutine[Any, Any, R]) -> R:
|
|
271
|
+
async with sem:
|
|
272
|
+
return await coro
|
|
273
|
+
|
|
274
|
+
async with asyncio.TaskGroup() as tg:
|
|
275
|
+
tasks = [tg.create_task(with_sem(c)) for c in coro_list]
|
|
276
|
+
|
|
277
|
+
for task in tasks:
|
|
278
|
+
yield task.result()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tracktolib
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.69.0
|
|
4
4
|
Summary: Utility library for python
|
|
5
5
|
Keywords: utility
|
|
6
6
|
Author-email: julien.brayere@tracktor.fr
|
|
@@ -12,6 +12,8 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.14
|
|
13
13
|
Requires-Dist: fastapi>=0.103.2 ; extra == 'api'
|
|
14
14
|
Requires-Dist: pydantic>=2 ; extra == 'api'
|
|
15
|
+
Requires-Dist: niquests>=3.17.0 ; extra == 'cf'
|
|
16
|
+
Requires-Dist: niquests>=3.17.0 ; extra == 'gh'
|
|
15
17
|
Requires-Dist: httpx>=0.25.0 ; extra == 'http'
|
|
16
18
|
Requires-Dist: python-json-logger>=3.2.1 ; extra == 'logs'
|
|
17
19
|
Requires-Dist: niquests>=3.17.0 ; extra == 'notion'
|
|
@@ -26,6 +28,8 @@ Requires-Dist: niquests>=3.17.0 ; extra == 's3-niquests'
|
|
|
26
28
|
Requires-Dist: deepdiff>=8.1.0 ; extra == 'tests'
|
|
27
29
|
Requires-Python: >=3.12, <4.0
|
|
28
30
|
Provides-Extra: api
|
|
31
|
+
Provides-Extra: cf
|
|
32
|
+
Provides-Extra: gh
|
|
29
33
|
Provides-Extra: http
|
|
30
34
|
Provides-Extra: logs
|
|
31
35
|
Provides-Extra: notion
|
|
@@ -126,12 +130,42 @@ uv add tracktolib[s3-minio]
|
|
|
126
130
|
|
|
127
131
|
### s3-niquests
|
|
128
132
|
|
|
129
|
-
Async S3 helpers using [niquests](https://github.com/jawah/niquests) and [botocore](https://github.com/boto/botocore).
|
|
133
|
+
Async S3 helpers using [niquests](https://github.com/jawah/niquests) and [botocore](https://github.com/boto/botocore) presigned URLs.
|
|
130
134
|
|
|
131
135
|
```bash
|
|
132
136
|
uv add tracktolib[s3-niquests]
|
|
133
137
|
```
|
|
134
138
|
|
|
139
|
+
```python
|
|
140
|
+
from tracktolib.s3.niquests import S3Session
|
|
141
|
+
|
|
142
|
+
async with S3Session(
|
|
143
|
+
endpoint_url='http://localhost:9000',
|
|
144
|
+
access_key='...',
|
|
145
|
+
secret_key='...',
|
|
146
|
+
region='us-east-1',
|
|
147
|
+
) as s3:
|
|
148
|
+
# Object operations
|
|
149
|
+
await s3.put_object('bucket', 'path/file.txt', b'content')
|
|
150
|
+
content = await s3.get_object('bucket', 'path/file.txt')
|
|
151
|
+
await s3.delete_object('bucket', 'path/file.txt')
|
|
152
|
+
|
|
153
|
+
# Streaming upload (multipart for large files)
|
|
154
|
+
async def data_stream():
|
|
155
|
+
yield b'chunk1'
|
|
156
|
+
yield b'chunk2'
|
|
157
|
+
await s3.file_upload('bucket', 'large-file.bin', data_stream())
|
|
158
|
+
|
|
159
|
+
# Bucket policy management
|
|
160
|
+
policy = {'Version': '2012-10-17', 'Statement': [...]}
|
|
161
|
+
await s3.put_bucket_policy('bucket', policy)
|
|
162
|
+
await s3.get_bucket_policy('bucket')
|
|
163
|
+
await s3.delete_bucket_policy('bucket')
|
|
164
|
+
|
|
165
|
+
# Empty a bucket (delete all objects)
|
|
166
|
+
deleted_count = await s3.empty_bucket('bucket')
|
|
167
|
+
```
|
|
168
|
+
|
|
135
169
|
### http (deprecated)
|
|
136
170
|
|
|
137
171
|
HTTP client helpers using [httpx](https://www.python-httpx.org/).
|
|
@@ -156,6 +190,85 @@ Notion API helpers using [niquests](https://github.com/jawah/niquests).
|
|
|
156
190
|
uv add tracktolib[notion]
|
|
157
191
|
```
|
|
158
192
|
|
|
193
|
+
```python
|
|
194
|
+
import niquests
|
|
195
|
+
from tracktolib.notion.fetch import fetch_database, get_notion_headers
|
|
196
|
+
from tracktolib.notion.cache import NotionCache
|
|
197
|
+
|
|
198
|
+
async with niquests.AsyncSession() as session:
|
|
199
|
+
session.headers.update(get_notion_headers())
|
|
200
|
+
|
|
201
|
+
# Without cache
|
|
202
|
+
db = await fetch_database(session, "database-id")
|
|
203
|
+
|
|
204
|
+
# With persistent cache (stored in ~/.cache/tracktolib/notion/cache.json)
|
|
205
|
+
cache = NotionCache()
|
|
206
|
+
db = await fetch_database(session, "database-id", cache=cache)
|
|
207
|
+
|
|
208
|
+
# Check cached databases
|
|
209
|
+
cache.get_databases() # All cached databases
|
|
210
|
+
cache.get_database("db-id") # Specific database (id, title, properties, cached_at)
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
### gh
|
|
214
|
+
|
|
215
|
+
GitHub API helpers using [niquests](https://github.com/jawah/niquests).
|
|
216
|
+
|
|
217
|
+
```bash
|
|
218
|
+
uv add tracktolib[gh]
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
```python
|
|
222
|
+
from tracktolib.gh import GitHubClient
|
|
223
|
+
|
|
224
|
+
async with GitHubClient() as gh: # Uses GITHUB_TOKEN env var
|
|
225
|
+
# Issue comments
|
|
226
|
+
comments = await gh.get_issue_comments("owner/repo", 123)
|
|
227
|
+
await gh.create_issue_comment("owner/repo", 123, "Hello!")
|
|
228
|
+
await gh.delete_comments_with_marker("owner/repo", 123, "<!-- bot -->")
|
|
229
|
+
|
|
230
|
+
# Labels
|
|
231
|
+
labels = await gh.get_issue_labels("owner/repo", 123)
|
|
232
|
+
await gh.add_labels("owner/repo", 123, ["bug", "priority"])
|
|
233
|
+
await gh.remove_label("owner/repo", 123, "wontfix")
|
|
234
|
+
|
|
235
|
+
# Deployments
|
|
236
|
+
deploys = await gh.get_deployments("owner/repo", environment="production")
|
|
237
|
+
await gh.mark_deployment_inactive("owner/repo", "preview-123")
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
### cf
|
|
241
|
+
|
|
242
|
+
Cloudflare DNS API helpers using [niquests](https://github.com/jawah/niquests).
|
|
243
|
+
|
|
244
|
+
```bash
|
|
245
|
+
uv add tracktolib[cf]
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
```python
|
|
249
|
+
from tracktolib.cf import CloudflareDNSClient
|
|
250
|
+
|
|
251
|
+
async with CloudflareDNSClient() as cf: # Uses CLOUDFLARE_API_TOKEN and CLOUDFLARE_ZONE_ID env vars
|
|
252
|
+
# Get a DNS record
|
|
253
|
+
record = await cf.get_dns_record("app.example.com", "CNAME")
|
|
254
|
+
|
|
255
|
+
# Create a DNS record
|
|
256
|
+
record = await cf.create_dns_record(
|
|
257
|
+
"app.example.com",
|
|
258
|
+
"target.example.com",
|
|
259
|
+
record_type="CNAME",
|
|
260
|
+
ttl=60,
|
|
261
|
+
proxied=True,
|
|
262
|
+
)
|
|
263
|
+
|
|
264
|
+
# Delete by ID or name
|
|
265
|
+
await cf.delete_dns_record(record["id"])
|
|
266
|
+
await cf.delete_dns_record_by_name("app.example.com", "CNAME")
|
|
267
|
+
|
|
268
|
+
# Check existence
|
|
269
|
+
exists = await cf.dns_record_exists("app.example.com")
|
|
270
|
+
```
|
|
271
|
+
|
|
159
272
|
### tests
|
|
160
273
|
|
|
161
274
|
Testing utilities using [deepdiff](https://github.com/seperman/deepdiff).
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
tracktolib/__init__.py,sha256=Q9d6h2lNjcYzxvfJ3zlNcpiP_Ak0T3TBPWINzZNrhu0,173
|
|
2
|
+
tracktolib/api.py,sha256=B212xUA8mcx6qWSAipZjKZq-i6wrU_1ZlqwQ9wrKTSs,10356
|
|
3
|
+
tracktolib/cf/__init__.py,sha256=_lT0T0uP9RRfTHBRLH43o5wxpuq9gxtKxSyyCYQuDmQ,168
|
|
4
|
+
tracktolib/cf/client.py,sha256=dUGCnpaKmb9qLOkyAlvCVY74bExqp2BAPmZvRK7JGuw,4951
|
|
5
|
+
tracktolib/cf/types.py,sha256=axRTBT_YvAYWYUACFKkcjcKIx61bRvGKkOhkVbG-dJI,366
|
|
6
|
+
tracktolib/gh/__init__.py,sha256=Yi2hecsG3ldbmQlMMsPizrZnIpDVpWA7DbzpcU6iBrI,254
|
|
7
|
+
tracktolib/gh/client.py,sha256=KL6Ndg9HiSa30naXq_bBh65soNhLKKfOfFpb4zjGMO4,8185
|
|
8
|
+
tracktolib/gh/types.py,sha256=sDdHTbbml7rdFFrb0SmOIB2ldTqQT-l53949EP9MANc,4491
|
|
9
|
+
tracktolib/http_utils.py,sha256=wek9FrZ_2yJbp0cWui3URqa8Iw2QRmugWanQWjx3RqQ,2785
|
|
10
|
+
tracktolib/logs.py,sha256=W9v4fcVuct2Ky2j1qM7IuYmyhOMNE6M4uGTGxt6fJCA,2191
|
|
11
|
+
tracktolib/notion/__init__.py,sha256=I-RAhMOCLvSDHyuKPVvbWSMX01qGcP7abun0NlgQZhM,1006
|
|
12
|
+
tracktolib/notion/blocks.py,sha256=IL-C8_eaRcMW0TQ736VgRKD84WQqNepi3UJq2s1lmzQ,12210
|
|
13
|
+
tracktolib/notion/cache.py,sha256=szOLoXlrw0t_6Oaz0k9HWxN7GtvJKfFiJpyZatq-hnc,6432
|
|
14
|
+
tracktolib/notion/fetch.py,sha256=Jw1KNNXbYeXCf03PGt9v5HeC_l55Fzf-q9cVr7_zhbg,16765
|
|
15
|
+
tracktolib/notion/markdown.py,sha256=vju-8TTrI8Fc0WzffudO-4R4ziknlEBMMo5v2dK0_IY,14835
|
|
16
|
+
tracktolib/notion/models.py,sha256=qYvgYzQz5nKsTf3kcbGjJRleiops11Na9Ay3OKxhQ5c,6107
|
|
17
|
+
tracktolib/notion/utils.py,sha256=7WltrDOa5ZLo2UCM4zeXyGd4SMt3NGmq-aBY4-PWgug,18910
|
|
18
|
+
tracktolib/pg/__init__.py,sha256=fPb25hSnUpG-7yNHgOjFpLTjbeOLkmU2tjczR371cc0,366
|
|
19
|
+
tracktolib/pg/query.py,sha256=4uG9BiJf91OxQI1kfizdN4JV1Vm81tOaKlJ-yUjXs2g,19346
|
|
20
|
+
tracktolib/pg/utils.py,sha256=ijI_gFzFSohy-BSBIuAxpBTeoK5L5lu2_f19H3E2p40,6296
|
|
21
|
+
tracktolib/pg_sync.py,sha256=b3QkhukkB8Yq6rmvH3xhUepRs9Tf0cvXXLyOJGDz4mg,6760
|
|
22
|
+
tracktolib/pg_utils.py,sha256=DRe5M0l7axTOPxHjDvVDBE0UnRlmESr7gTq-IaLOETc,2534
|
|
23
|
+
tracktolib/s3/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
24
|
+
tracktolib/s3/minio.py,sha256=KJTNJbdmlZwhnMm40-Imc5M1MDY_A3_p-l6ltc_lHGc,1387
|
|
25
|
+
tracktolib/s3/niquests.py,sha256=Z7FRo_eOAKeKNg9UrnsMYXAiYcCKS-_rvkgyZnPS1x8,31995
|
|
26
|
+
tracktolib/s3/s3.py,sha256=zC72K209z0CIijTCtKtapQbOZqja-U1FCeuY8ipDRHQ,4922
|
|
27
|
+
tracktolib/tests.py,sha256=gKE--epQjgMZGXc5ydbl4zjOdmwztJS42UMV0p4hXEA,399
|
|
28
|
+
tracktolib/utils.py,sha256=k62RR8Qukse9Ci9ZYejLsmpFBbClTl4116-TH_EP2Gs,7520
|
|
29
|
+
tracktolib-0.69.0.dist-info/WHEEL,sha256=5DEXXimM34_d4Gx1AuF9ysMr1_maoEtGKjaILM3s4w4,80
|
|
30
|
+
tracktolib-0.69.0.dist-info/METADATA,sha256=nD-YW_COpZkQ6Nb04W8NXU9uzXLHQg0flvPU6DUOQdM,7396
|
|
31
|
+
tracktolib-0.69.0.dist-info/RECORD,,
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
tracktolib/__init__.py,sha256=Q9d6h2lNjcYzxvfJ3zlNcpiP_Ak0T3TBPWINzZNrhu0,173
|
|
2
|
-
tracktolib/api.py,sha256=-TepGdrKH7SAvQBEuSt49aE5-XaSHIX9ugUActEwgqY,10389
|
|
3
|
-
tracktolib/http_utils.py,sha256=_PJlvmKBwaJAGOWYnwU4LP_yV3oaMCk9nrI1u2iFBuk,2785
|
|
4
|
-
tracktolib/logs.py,sha256=D2hx6urXl5l4PBGP8mCpcT4GX7tJeFfNY-7oBfHczBU,2191
|
|
5
|
-
tracktolib/notion/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
|
-
tracktolib/notion/fetch.py,sha256=fQw42gab5eDfphmtzeRYZKWFLFZztjxTLgX5xfc46Pc,13437
|
|
7
|
-
tracktolib/notion/models.py,sha256=FbTJcK0eA-4phpfjUxyAW7cs5jHZQxB6qqZ75ZcJ7uw,5608
|
|
8
|
-
tracktolib/pg/__init__.py,sha256=Ul_hgwvTXZvQBt7sHKi4ZI-0DDpnXmoFtmVkGRy-1J0,366
|
|
9
|
-
tracktolib/pg/query.py,sha256=Sarwvs8cSqiOQLUnpTOx2XsDClr0dKACPvQfTl_v8_Y,19346
|
|
10
|
-
tracktolib/pg/utils.py,sha256=ygQn63EBDaEGB0p7P2ibellO2mv-StafanpXKcCUiZU,6324
|
|
11
|
-
tracktolib/pg_sync.py,sha256=PDTN37kU0BxkSZetwSAtqcW2aA8Nn4gUI2mC54gSJhg,6750
|
|
12
|
-
tracktolib/pg_utils.py,sha256=ArYNdf9qsdYdzGEWmev8tZpyx8_1jaGGdkfYkauM7UM,2582
|
|
13
|
-
tracktolib/s3/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
14
|
-
tracktolib/s3/minio.py,sha256=wMEjkSes9Fp39fD17IctALpD6zB2xwDRQEmO7Vzan3g,1387
|
|
15
|
-
tracktolib/s3/niquests.py,sha256=9j3RxM3EfIYV1wEH0OpvT_uhJ68sXN4PwxlDAH3eBEE,23453
|
|
16
|
-
tracktolib/s3/s3.py,sha256=Vi3Q6DLBm44gz6fXx6uzdbGEtJly6KzdgLYHJwU6r-U,4922
|
|
17
|
-
tracktolib/tests.py,sha256=gKE--epQjgMZGXc5ydbl4zjOdmwztJS42UMV0p4hXEA,399
|
|
18
|
-
tracktolib/utils.py,sha256=FP87gbL27zHXaI9My2VZYEG5ZJ7eL6SiljW5MyRutOY,6553
|
|
19
|
-
tracktolib-0.67.0.dist-info/WHEEL,sha256=XV0cjMrO7zXhVAIyyc8aFf1VjZ33Fen4IiJk5zFlC3g,80
|
|
20
|
-
tracktolib-0.67.0.dist-info/METADATA,sha256=yJodZh9-6H2mIOQcQWguUE9zsbDqOPKLyTh3A_tHUAo,4045
|
|
21
|
-
tracktolib-0.67.0.dist-info/RECORD,,
|