plexus-python-pulse-common 1.1.91__py3-none-any.whl → 1.1.92__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- plexus/pulse/common/utils/apiutils.py +19 -1
- plexus/pulse/common/utils/s3utils.py +138 -3
- plexus/pulse/common/utils/tagutils.py +5 -6
- {plexus_python_pulse_common-1.1.91.dist-info → plexus_python_pulse_common-1.1.92.dist-info}/METADATA +1 -1
- {plexus_python_pulse_common-1.1.91.dist-info → plexus_python_pulse_common-1.1.92.dist-info}/RECORD +7 -8
- plexus/pulse/common/utils/sqlutils.py +0 -9
- {plexus_python_pulse_common-1.1.91.dist-info → plexus_python_pulse_common-1.1.92.dist-info}/WHEEL +0 -0
- {plexus_python_pulse_common-1.1.91.dist-info → plexus_python_pulse_common-1.1.92.dist-info}/top_level.txt +0 -0
|
@@ -1,9 +1,13 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
import uuid as py_uuid
|
|
1
3
|
from collections.abc import Callable, Mapping
|
|
2
4
|
from contextlib import AbstractContextManager
|
|
3
5
|
|
|
6
|
+
import sqlalchemy as sa
|
|
4
7
|
import sqlalchemy.orm as sa_orm
|
|
5
8
|
from fastapi import HTTPException
|
|
6
9
|
from plexus.common.utils import logger
|
|
10
|
+
from sqlmodel import SQLModel
|
|
7
11
|
|
|
8
12
|
__all__ = [
|
|
9
13
|
"database_transaction",
|
|
@@ -25,7 +29,7 @@ class DBTransactionManager(AbstractContextManager):
|
|
|
25
29
|
def __enter__(self):
|
|
26
30
|
return self
|
|
27
31
|
|
|
28
|
-
def __exit__(self, exc_type, exc_value, traceback):
|
|
32
|
+
def __exit__(self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback):
|
|
29
33
|
if exc_type is None:
|
|
30
34
|
if self.commit_on_exit:
|
|
31
35
|
try:
|
|
@@ -62,3 +66,17 @@ def database_transaction(
|
|
|
62
66
|
redirect: Mapping[type[Exception], Callable[[Exception], HTTPException]] | None = None,
|
|
63
67
|
) -> DBTransactionManager:
|
|
64
68
|
return DBTransactionManager(db, commit_on_exit=commit_on_exit, redirect=redirect)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def get_clock() -> Callable[[sa_orm.Session], datetime.datetime]:
|
|
72
|
+
def database_now(db: sa_orm.Session) -> datetime.datetime:
|
|
73
|
+
return db.scalar(sa.select(sa.func.now()))
|
|
74
|
+
|
|
75
|
+
return database_now
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def get_uuid_generator() -> Callable[[type[SQLModel]], py_uuid.UUID]:
|
|
79
|
+
def uuid_generator(model: type[SQLModel]) -> py_uuid.UUID:
|
|
80
|
+
return py_uuid.uuid4()
|
|
81
|
+
|
|
82
|
+
return uuid_generator
|
|
@@ -13,11 +13,12 @@ import typing
|
|
|
13
13
|
import zipfile
|
|
14
14
|
import zlib
|
|
15
15
|
from collections.abc import Callable, Generator
|
|
16
|
-
from typing import Literal
|
|
16
|
+
from typing import Any, Literal, Self
|
|
17
17
|
|
|
18
18
|
import boto3
|
|
19
19
|
import fsspec
|
|
20
20
|
import fsspec.utils
|
|
21
|
+
import pydantic as pdt
|
|
21
22
|
from mypy_boto3_s3 import S3Client
|
|
22
23
|
from plexus.common.utils.iterutils import chunk_between, head, last
|
|
23
24
|
from plexus.common.utils.jsonutils import JsonObject
|
|
@@ -26,6 +27,11 @@ from plexus.common.utils.strutils import is_empty, trim_to_none
|
|
|
26
27
|
from rich.progress import BarColumn, DownloadColumn, Progress, TextColumn, TransferSpeedColumn
|
|
27
28
|
|
|
28
29
|
__all__ = [
|
|
30
|
+
"S3UriLike",
|
|
31
|
+
"S3Uri",
|
|
32
|
+
"S3UriMapping",
|
|
33
|
+
"S3UriMappingConfig",
|
|
34
|
+
"S3UriReinterpreter",
|
|
29
35
|
"S3ObjectMeta",
|
|
30
36
|
"s3_make_client",
|
|
31
37
|
"s3_head_object",
|
|
@@ -48,8 +54,137 @@ __all__ = [
|
|
|
48
54
|
]
|
|
49
55
|
|
|
50
56
|
|
|
51
|
-
@
|
|
52
|
-
class
|
|
57
|
+
@typing.runtime_checkable
|
|
58
|
+
class S3UriLike(typing.Protocol):
|
|
59
|
+
"""A path object that can expose its full URI, such as ``upath.UPath``."""
|
|
60
|
+
|
|
61
|
+
def as_uri(self) -> str: ...
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class S3Uri(pdt.BaseModel):
|
|
65
|
+
model_config = pdt.ConfigDict(strict=True, extra="forbid", frozen=True)
|
|
66
|
+
|
|
67
|
+
protocol: str
|
|
68
|
+
bucket: str
|
|
69
|
+
key: str = ""
|
|
70
|
+
|
|
71
|
+
@classmethod
|
|
72
|
+
def parse(cls, uri: str) -> Self:
|
|
73
|
+
return cls.model_validate(uri)
|
|
74
|
+
|
|
75
|
+
@pdt.model_validator(mode="before")
|
|
76
|
+
@classmethod
|
|
77
|
+
def parse_uri(cls, value: Any) -> Any:
|
|
78
|
+
if not isinstance(value, str):
|
|
79
|
+
return value
|
|
80
|
+
|
|
81
|
+
fs_opts = fsspec.utils.infer_storage_options(value)
|
|
82
|
+
|
|
83
|
+
protocol = fs_opts.get("protocol")
|
|
84
|
+
bucket = fs_opts.get("host")
|
|
85
|
+
_, separator, remainder = value.partition("://")
|
|
86
|
+
authority, _, key = remainder.partition("/")
|
|
87
|
+
|
|
88
|
+
if not separator or not protocol or not bucket or authority != bucket:
|
|
89
|
+
raise ValueError(f"invalid S3 URI '{value!r}'")
|
|
90
|
+
|
|
91
|
+
for segment in {"username", "password", "port", "url_query", "url_fragment"}:
|
|
92
|
+
if segment in fs_opts:
|
|
93
|
+
raise ValueError(f"unexpected segment {segment} in S3 URI '{value!r}'")
|
|
94
|
+
|
|
95
|
+
return {"protocol": protocol, "bucket": bucket, "key": key}
|
|
96
|
+
|
|
97
|
+
@pdt.model_serializer
|
|
98
|
+
def serialize_uri(self) -> str:
|
|
99
|
+
return str(self)
|
|
100
|
+
|
|
101
|
+
def as_uri(self) -> str:
|
|
102
|
+
return f"{self.protocol}://{self.bucket}/{self.key}"
|
|
103
|
+
|
|
104
|
+
def __str__(self) -> str:
|
|
105
|
+
return self.as_uri()
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class S3UriMapping(pdt.BaseModel):
|
|
109
|
+
model_config = pdt.ConfigDict(strict=True, extra="forbid", frozen=True)
|
|
110
|
+
|
|
111
|
+
source: S3Uri
|
|
112
|
+
target: S3Uri
|
|
113
|
+
|
|
114
|
+
@property
|
|
115
|
+
def specificity(self) -> int:
|
|
116
|
+
return len(self.source.key.rstrip("/"))
|
|
117
|
+
|
|
118
|
+
def apply(self, uri: S3Uri) -> S3Uri | None:
|
|
119
|
+
if (uri.protocol, uri.bucket) != (self.source.protocol, self.source.bucket):
|
|
120
|
+
return None
|
|
121
|
+
|
|
122
|
+
source = self.source.key.rstrip("/")
|
|
123
|
+
target = self.target.key.rstrip("/")
|
|
124
|
+
|
|
125
|
+
if not source:
|
|
126
|
+
key = target + "/" + uri.key if target and uri.key else target or uri.key
|
|
127
|
+
elif uri.key == source:
|
|
128
|
+
key = self.target.key
|
|
129
|
+
elif uri.key.startswith(source + "/"):
|
|
130
|
+
suffix = uri.key[len(source):]
|
|
131
|
+
key = target + suffix if target else suffix[1:]
|
|
132
|
+
else:
|
|
133
|
+
return None
|
|
134
|
+
|
|
135
|
+
return S3Uri(protocol=self.target.protocol, bucket=self.target.bucket, key=key)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class S3UriMappingConfig(pdt.BaseModel):
|
|
139
|
+
model_config = pdt.ConfigDict(strict=True, extra="forbid")
|
|
140
|
+
|
|
141
|
+
mappings: list[S3UriMapping] = pdt.Field(default_factory=list)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class S3UriReinterpreter:
|
|
145
|
+
def __init__(self, config: S3UriMappingConfig, *, strict: bool = False):
|
|
146
|
+
|
|
147
|
+
sources = [(mapping.source.protocol, mapping.source.bucket, mapping.source.key.rstrip("/"))
|
|
148
|
+
for mapping in config.mappings]
|
|
149
|
+
|
|
150
|
+
if len(sources) != len(set(sources)):
|
|
151
|
+
raise ValueError("duplicate S3 mapping source")
|
|
152
|
+
|
|
153
|
+
self.mappings = sorted(config.mappings, key=lambda m: m.specificity, reverse=True)
|
|
154
|
+
self.strict = strict
|
|
155
|
+
|
|
156
|
+
@classmethod
|
|
157
|
+
def from_config(cls, config: JsonObject | S3UriMappingConfig, strict: bool = False) -> Self:
|
|
158
|
+
"""Create from an already loaded and selected configuration."""
|
|
159
|
+
return cls(S3UriMappingConfig.model_validate(config), strict=strict)
|
|
160
|
+
|
|
161
|
+
def reinterpret(self, uri: str | os.PathLike[str] | S3UriLike) -> S3Uri:
|
|
162
|
+
"""Resolve a URI value or a path-like object that preserves URI syntax.
|
|
163
|
+
|
|
164
|
+
``pathlib.Path`` normalizes ``s3://`` to ``s3:/``, so construct S3
|
|
165
|
+
locations with strings or ``S3Uri`` instead.
|
|
166
|
+
"""
|
|
167
|
+
if isinstance(uri, S3Uri):
|
|
168
|
+
parsed = uri
|
|
169
|
+
elif isinstance(uri, S3UriLike):
|
|
170
|
+
parsed = S3Uri.parse(uri.as_uri())
|
|
171
|
+
else:
|
|
172
|
+
parsed = S3Uri.parse(os.fspath(uri))
|
|
173
|
+
|
|
174
|
+
for mapping in self.mappings:
|
|
175
|
+
result = mapping.apply(parsed)
|
|
176
|
+
if result is not None:
|
|
177
|
+
return result
|
|
178
|
+
|
|
179
|
+
if self.strict:
|
|
180
|
+
raise ValueError(f"no mapping for '{uri!r}'")
|
|
181
|
+
|
|
182
|
+
return parsed
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
class S3ObjectMeta(pdt.BaseModel):
|
|
186
|
+
model_config = pdt.ConfigDict(strict=True, extra="forbid", frozen=True)
|
|
187
|
+
|
|
53
188
|
key: str
|
|
54
189
|
last_modified: datetime.datetime
|
|
55
190
|
size: int
|
|
@@ -35,7 +35,6 @@ from plexus.pulse.common.utils.datautils import validate_dt_timezone, validate_s
|
|
|
35
35
|
from plexus.pulse.common.utils.jsonutils import json_datetime_encoder
|
|
36
36
|
from plexus.pulse.common.utils.ormutils import SQLiteDateTime, SequenceModelMixinProtocol
|
|
37
37
|
from plexus.pulse.common.utils.ormutils import clone_sequence_model_instance, make_base_model, make_sequence_model_mixin
|
|
38
|
-
from plexus.pulse.common.utils.sqlutils import escape_sql_like
|
|
39
38
|
|
|
40
39
|
__all__ = [
|
|
41
40
|
"versioned_identifier",
|
|
@@ -958,7 +957,7 @@ class TagCache(object):
|
|
|
958
957
|
if tagset_version:
|
|
959
958
|
query_stmt = query_stmt.filter(TagRecordTable.tagset_version == tagset_version)
|
|
960
959
|
if tag_prefix:
|
|
961
|
-
query_stmt = query_stmt.filter(TagRecordTable.tag.
|
|
960
|
+
query_stmt = query_stmt.filter(TagRecordTable.tag.startswith(tag_prefix, autoescape=True))
|
|
962
961
|
if tagsets:
|
|
963
962
|
if tagset_inverted:
|
|
964
963
|
query_stmt = query_stmt.filter(
|
|
@@ -1033,7 +1032,7 @@ class TagCache(object):
|
|
|
1033
1032
|
if tagset_version:
|
|
1034
1033
|
query_stmt = query_stmt.filter(TagRecordTable.tagset_version == tagset_version)
|
|
1035
1034
|
if tag_prefix:
|
|
1036
|
-
query_stmt = query_stmt.filter(TagRecordTable.tag.
|
|
1035
|
+
query_stmt = query_stmt.filter(TagRecordTable.tag.startswith(tag_prefix, autoescape=True))
|
|
1037
1036
|
if target_identifier:
|
|
1038
1037
|
query_stmt = query_stmt.filter(TagTargetTable.identifier == target_identifier)
|
|
1039
1038
|
if target_tagger_name:
|
|
@@ -1098,7 +1097,7 @@ class TagCache(object):
|
|
|
1098
1097
|
if tagset_version:
|
|
1099
1098
|
query_stmt = query_stmt.filter(TagRecordTable.tagset_version == tagset_version)
|
|
1100
1099
|
if tag_prefix:
|
|
1101
|
-
query_stmt = query_stmt.filter(TagRecordTable.tag.
|
|
1100
|
+
query_stmt = query_stmt.filter(TagRecordTable.tag.startswith(tag_prefix, autoescape=True))
|
|
1102
1101
|
if tagsets:
|
|
1103
1102
|
if tagset_inverted:
|
|
1104
1103
|
query_stmt = query_stmt.filter(
|
|
@@ -1165,7 +1164,7 @@ class TagCache(object):
|
|
|
1165
1164
|
if tagset_version:
|
|
1166
1165
|
query_stmt = query_stmt.filter(TagRecordTable.tagset_version == tagset_version)
|
|
1167
1166
|
if tag_prefix:
|
|
1168
|
-
query_stmt = query_stmt.filter(TagRecordTable.tag.
|
|
1167
|
+
query_stmt = query_stmt.filter(TagRecordTable.tag.startswith(tag_prefix, autoescape=True))
|
|
1169
1168
|
if tagsets:
|
|
1170
1169
|
if tagset_inverted:
|
|
1171
1170
|
query_stmt = query_stmt.filter(
|
|
@@ -1362,7 +1361,7 @@ class TagCache(object):
|
|
|
1362
1361
|
if tagset_version:
|
|
1363
1362
|
query_stmt = query_stmt.filter(TagRecordTable.tagset_version == tagset_version)
|
|
1364
1363
|
if tag_prefix:
|
|
1365
|
-
query_stmt = query_stmt.filter(TagRecordTable.tag.
|
|
1364
|
+
query_stmt = query_stmt.filter(TagRecordTable.tag.startswith(tag_prefix, autoescape=True))
|
|
1366
1365
|
if tagsets:
|
|
1367
1366
|
if tagset_inverted:
|
|
1368
1367
|
query_stmt = query_stmt.filter(
|
{plexus_python_pulse_common-1.1.91.dist-info → plexus_python_pulse_common-1.1.92.dist-info}/RECORD
RENAMED
|
@@ -9,7 +9,7 @@ plexus/pulse/common/resources/tags/__init__.py,sha256=1BmOq9aIT2Fr4YTzDwTrYySDhi
|
|
|
9
9
|
plexus/pulse/common/resources/tags/unittest-1.0.0.tagset.yaml,sha256=8zWjsYp3OxGfBJSiS6-r08q61Jhulol65iV5-pvF1-M,5338
|
|
10
10
|
plexus/pulse/common/resources/tags/universal-1.0.0.tagset.yaml,sha256=PRxyS5xwWcNwUjAbYqN-SafLZVviE-QSzU-DXyXNffw,80736
|
|
11
11
|
plexus/pulse/common/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
|
-
plexus/pulse/common/utils/apiutils.py,sha256=
|
|
12
|
+
plexus/pulse/common/utils/apiutils.py,sha256=03efswisR-XPmXqe29dWKB8Cl6cyaRue1VgLVPsfqDA,2616
|
|
13
13
|
plexus/pulse/common/utils/bagutils.py,sha256=JcTD-9xJAhV_BYXw-Zfa_85hkyO5wUVqTqw2eMJFgVI,13116
|
|
14
14
|
plexus/pulse/common/utils/config.py,sha256=YPtFK5QHTVsXwcFBGV0rlZad0GFtsgwysQNyKYjdzvk,2409
|
|
15
15
|
plexus/pulse/common/utils/datautils.py,sha256=OWlnyFSa6UaPbSSG3HzCJHxdiHyfHAE-FUVHaLfNZoM,9335
|
|
@@ -19,12 +19,11 @@ plexus/pulse/common/utils/gisutils.py,sha256=Cx0ZB9-NA_7VuzbuzRArTJ5UfltG-KmvVC8
|
|
|
19
19
|
plexus/pulse/common/utils/jsonutils.py,sha256=zSbW94klN2jrkMdBECu4RNJX6zzNaSxauQ3FuihuB6E,3342
|
|
20
20
|
plexus/pulse/common/utils/ormutils.py,sha256=taaZzpIH0C-ShEmOWf-vt22bgcFFqH1FnJlNcH9tEFM,68222
|
|
21
21
|
plexus/pulse/common/utils/pathutils.py,sha256=eYfNK48wf1xZGK2IbpKxYOn4skZhWWA8OTyUwDXGMIc,9222
|
|
22
|
-
plexus/pulse/common/utils/s3utils.py,sha256=
|
|
23
|
-
plexus/pulse/common/utils/sqlutils.py,sha256=D6kTBjhO5YlNRt3uFlPt6z3uH61m9ajEzPYmsI6NoFc,231
|
|
22
|
+
plexus/pulse/common/utils/s3utils.py,sha256=MDQHJCLbnkCnvyzBCIsK-IRqloy91Sv7be2y4NsoryE,40219
|
|
24
23
|
plexus/pulse/common/utils/strutils.py,sha256=8kl88-u89FApoaq-D2ulk5DNl9kenP6vqksvVJ0iH4s,17325
|
|
25
|
-
plexus/pulse/common/utils/tagutils.py,sha256=
|
|
24
|
+
plexus/pulse/common/utils/tagutils.py,sha256=BG8E87TBu4zzx17mb-219f6M5OtdJknqgHf2WZDJhHQ,66816
|
|
26
25
|
plexus/pulse/common/utils/testutils.py,sha256=hKPRvoN4mg8QGQJH121l59a0uo8jN5rntGQAVYplGBw,6166
|
|
27
|
-
plexus_python_pulse_common-1.1.
|
|
28
|
-
plexus_python_pulse_common-1.1.
|
|
29
|
-
plexus_python_pulse_common-1.1.
|
|
30
|
-
plexus_python_pulse_common-1.1.
|
|
26
|
+
plexus_python_pulse_common-1.1.92.dist-info/METADATA,sha256=okfRgpKW4vaPBTb61oS4aeFtaKNhdTz6DaApIT1SwE4,1556
|
|
27
|
+
plexus_python_pulse_common-1.1.92.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
28
|
+
plexus_python_pulse_common-1.1.92.dist-info/top_level.txt,sha256=ug_g7CVwaMQuas5UzAXbHUrQvKGCn8ezc6ZNvvRlJOE,7
|
|
29
|
+
plexus_python_pulse_common-1.1.92.dist-info/RECORD,,
|
{plexus_python_pulse_common-1.1.91.dist-info → plexus_python_pulse_common-1.1.92.dist-info}/WHEEL
RENAMED
|
File without changes
|
|
File without changes
|