plexus-python-pulse-common 1.1.91__tar.gz → 1.1.92__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/PKG-INFO +1 -1
  2. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/utils/apiutils.py +19 -1
  3. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/utils/s3utils.py +138 -3
  4. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/utils/tagutils.py +5 -6
  5. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus_python_pulse_common.egg-info/PKG-INFO +1 -1
  6. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus_python_pulse_common.egg-info/SOURCES.txt +0 -1
  7. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/test/plexus_tests/pulse/common/utils/s3utils_test.py +157 -0
  8. plexus_python_pulse_common-1.1.91/src/plexus/pulse/common/utils/sqlutils.py +0 -9
  9. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/.editorconfig +0 -0
  10. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/.github/workflows/pr.yml +0 -0
  11. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/.github/workflows/push.yml +0 -0
  12. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/.gitignore +0 -0
  13. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/MANIFEST.in +0 -0
  14. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/README.md +0 -0
  15. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/VERSION +0 -0
  16. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/pyproject.toml +0 -0
  17. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/jsonutils/dummy.0.jsonl +0 -0
  18. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/jsonutils/dummy.1.jsonl +0 -0
  19. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/jsonutils/dummy.2.jsonl +0 -0
  20. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/pathutils/0-dummy +0 -0
  21. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/pathutils/1-dummy +0 -0
  22. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/pathutils/2-dummy +0 -0
  23. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/pathutils/dummy.0.0.jsonl +0 -0
  24. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/pathutils/dummy.0.0.vol-0.jsonl +0 -0
  25. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/pathutils/dummy.0.jsonl +0 -0
  26. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/pathutils/dummy.1.1.jsonl +0 -0
  27. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/pathutils/dummy.1.1.vol-1.jsonl +0 -0
  28. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/pathutils/dummy.1.jsonl +0 -0
  29. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/pathutils/dummy.2.2.jsonl +0 -0
  30. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/pathutils/dummy.2.2.vol-2.jsonl +0 -0
  31. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/pathutils/dummy.2.jsonl +0 -0
  32. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/pathutils/dummy.csv.part0 +0 -0
  33. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/pathutils/dummy.csv.part1 +0 -0
  34. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/pathutils/dummy.csv.part2 +0 -0
  35. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/pathutils/dummy.txt +0 -0
  36. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/s3utils/dir.baz/file.bar.baz +0 -0
  37. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/s3utils/dir.baz/file.foo.bar +0 -0
  38. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/s3utils/dir.baz/file.foo.baz +0 -0
  39. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/s3utils/dir.foo/dir.foo.bar/dir.foo.bar.baz/file.foo.bar.baz +0 -0
  40. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/s3utils/dir.foo/dir.foo.bar/file.bar.baz +0 -0
  41. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/s3utils/dir.foo/dir.foo.bar/file.foo.bar +0 -0
  42. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/s3utils/dir.foo/dir.foo.bar/file.foo.baz +0 -0
  43. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/s3utils/dir.foo/file.bar +0 -0
  44. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/s3utils/dir.foo/file.baz +0 -0
  45. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/s3utils/dir.foo/file.foo +0 -0
  46. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/s3utils_archive/archive.compressed.zip +0 -0
  47. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/resources/unittest/s3utils_archive/archive.uncompressed.zip +0 -0
  48. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/setup.cfg +0 -0
  49. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/setup.py +0 -0
  50. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/__init__.py +0 -0
  51. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/carto/OSMFile.py +0 -0
  52. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/carto/OSMNode.py +0 -0
  53. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/carto/OSMTags.py +0 -0
  54. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/carto/OSMWay.py +0 -0
  55. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/carto/__init__.py +0 -0
  56. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/resources/__init__.py +0 -0
  57. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/resources/tags/__init__.py +0 -0
  58. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/resources/tags/unittest-1.0.0.tagset.yaml +0 -0
  59. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/resources/tags/universal-1.0.0.tagset.yaml +0 -0
  60. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/utils/__init__.py +0 -0
  61. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/utils/bagutils.py +0 -0
  62. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/utils/config.py +0 -0
  63. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/utils/datautils.py +0 -0
  64. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/utils/dockerutils.py +0 -0
  65. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/utils/flowutils.py +0 -0
  66. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/utils/gisutils.py +0 -0
  67. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/utils/jsonutils.py +0 -0
  68. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/utils/ormutils.py +0 -0
  69. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/utils/pathutils.py +0 -0
  70. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/utils/strutils.py +0 -0
  71. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus/pulse/common/utils/testutils.py +0 -0
  72. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus_python_pulse_common.egg-info/dependency_links.txt +0 -0
  73. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus_python_pulse_common.egg-info/not-zip-safe +0 -0
  74. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus_python_pulse_common.egg-info/requires.txt +0 -0
  75. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/src/plexus_python_pulse_common.egg-info/top_level.txt +0 -0
  76. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/test/plexus_tests/__init__.py +0 -0
  77. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/test/plexus_tests/pulse/__init__.py +0 -0
  78. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/test/plexus_tests/pulse/common/__init__.py +0 -0
  79. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/test/plexus_tests/pulse/common/carto/__init__.py +0 -0
  80. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/test/plexus_tests/pulse/common/carto/osm_file_test.py +0 -0
  81. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/test/plexus_tests/pulse/common/carto/osm_tags_test.py +0 -0
  82. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/test/plexus_tests/pulse/common/utils/__init__.py +0 -0
  83. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/test/plexus_tests/pulse/common/utils/apiutils_test.py +0 -0
  84. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/test/plexus_tests/pulse/common/utils/bagutils_test.py +0 -0
  85. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/test/plexus_tests/pulse/common/utils/datautils_test.py +0 -0
  86. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/test/plexus_tests/pulse/common/utils/dockerutils_test.py +0 -0
  87. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/test/plexus_tests/pulse/common/utils/flowutils_test.py +0 -0
  88. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/test/plexus_tests/pulse/common/utils/gisutils_test.py +0 -0
  89. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/test/plexus_tests/pulse/common/utils/jsonutils_test.py +0 -0
  90. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/test/plexus_tests/pulse/common/utils/ormutils_test.py +0 -0
  91. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/test/plexus_tests/pulse/common/utils/pathutils_test.py +0 -0
  92. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/test/plexus_tests/pulse/common/utils/strutils_test.py +0 -0
  93. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/test/plexus_tests/pulse/common/utils/tagutils_test.py +0 -0
  94. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/test/plexus_tests/pulse/common/utils/testutils_test.py +0 -0
  95. {plexus_python_pulse_common-1.1.91 → plexus_python_pulse_common-1.1.92}/test/testenv.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: plexus-python-pulse-common
3
- Version: 1.1.91
3
+ Version: 1.1.92
4
4
  Classifier: Programming Language :: Python :: 3
5
5
  Classifier: Programming Language :: Python :: 3.12
6
6
  Classifier: Programming Language :: Python :: 3.13
@@ -1,9 +1,13 @@
1
+ import datetime
2
+ import uuid as py_uuid
1
3
  from collections.abc import Callable, Mapping
2
4
  from contextlib import AbstractContextManager
3
5
 
6
+ import sqlalchemy as sa
4
7
  import sqlalchemy.orm as sa_orm
5
8
  from fastapi import HTTPException
6
9
  from plexus.common.utils import logger
10
+ from sqlmodel import SQLModel
7
11
 
8
12
  __all__ = [
9
13
  "database_transaction",
@@ -25,7 +29,7 @@ class DBTransactionManager(AbstractContextManager):
25
29
  def __enter__(self):
26
30
  return self
27
31
 
28
- def __exit__(self, exc_type, exc_value, traceback):
32
+ def __exit__(self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback):
29
33
  if exc_type is None:
30
34
  if self.commit_on_exit:
31
35
  try:
@@ -62,3 +66,17 @@ def database_transaction(
62
66
  redirect: Mapping[type[Exception], Callable[[Exception], HTTPException]] | None = None,
63
67
  ) -> DBTransactionManager:
64
68
  return DBTransactionManager(db, commit_on_exit=commit_on_exit, redirect=redirect)
69
+
70
+
71
+ def get_clock() -> Callable[[sa_orm.Session], datetime.datetime]:
72
+ def database_now(db: sa_orm.Session) -> datetime.datetime:
73
+ return db.scalar(sa.select(sa.func.now()))
74
+
75
+ return database_now
76
+
77
+
78
+ def get_uuid_generator() -> Callable[[type[SQLModel]], py_uuid.UUID]:
79
+ def uuid_generator(model: type[SQLModel]) -> py_uuid.UUID:
80
+ return py_uuid.uuid4()
81
+
82
+ return uuid_generator
@@ -13,11 +13,12 @@ import typing
13
13
  import zipfile
14
14
  import zlib
15
15
  from collections.abc import Callable, Generator
16
- from typing import Literal
16
+ from typing import Any, Literal, Self
17
17
 
18
18
  import boto3
19
19
  import fsspec
20
20
  import fsspec.utils
21
+ import pydantic as pdt
21
22
  from mypy_boto3_s3 import S3Client
22
23
  from plexus.common.utils.iterutils import chunk_between, head, last
23
24
  from plexus.common.utils.jsonutils import JsonObject
@@ -26,6 +27,11 @@ from plexus.common.utils.strutils import is_empty, trim_to_none
26
27
  from rich.progress import BarColumn, DownloadColumn, Progress, TextColumn, TransferSpeedColumn
27
28
 
28
29
  __all__ = [
30
+ "S3UriLike",
31
+ "S3Uri",
32
+ "S3UriMapping",
33
+ "S3UriMappingConfig",
34
+ "S3UriReinterpreter",
29
35
  "S3ObjectMeta",
30
36
  "s3_make_client",
31
37
  "s3_head_object",
@@ -48,8 +54,137 @@ __all__ = [
48
54
  ]
49
55
 
50
56
 
51
- @dataclasses.dataclass(frozen=True, eq=True)
52
- class S3ObjectMeta(object):
57
+ @typing.runtime_checkable
58
+ class S3UriLike(typing.Protocol):
59
+ """A path object that can expose its full URI, such as ``upath.UPath``."""
60
+
61
+ def as_uri(self) -> str: ...
62
+
63
+
64
+ class S3Uri(pdt.BaseModel):
65
+ model_config = pdt.ConfigDict(strict=True, extra="forbid", frozen=True)
66
+
67
+ protocol: str
68
+ bucket: str
69
+ key: str = ""
70
+
71
+ @classmethod
72
+ def parse(cls, uri: str) -> Self:
73
+ return cls.model_validate(uri)
74
+
75
+ @pdt.model_validator(mode="before")
76
+ @classmethod
77
+ def parse_uri(cls, value: Any) -> Any:
78
+ if not isinstance(value, str):
79
+ return value
80
+
81
+ fs_opts = fsspec.utils.infer_storage_options(value)
82
+
83
+ protocol = fs_opts.get("protocol")
84
+ bucket = fs_opts.get("host")
85
+ _, separator, remainder = value.partition("://")
86
+ authority, _, key = remainder.partition("/")
87
+
88
+ if not separator or not protocol or not bucket or authority != bucket:
89
+ raise ValueError(f"invalid S3 URI '{value!r}'")
90
+
91
+ for segment in {"username", "password", "port", "url_query", "url_fragment"}:
92
+ if segment in fs_opts:
93
+ raise ValueError(f"unexpected segment {segment} in S3 URI '{value!r}'")
94
+
95
+ return {"protocol": protocol, "bucket": bucket, "key": key}
96
+
97
+ @pdt.model_serializer
98
+ def serialize_uri(self) -> str:
99
+ return str(self)
100
+
101
+ def as_uri(self) -> str:
102
+ return f"{self.protocol}://{self.bucket}/{self.key}"
103
+
104
+ def __str__(self) -> str:
105
+ return self.as_uri()
106
+
107
+
108
+ class S3UriMapping(pdt.BaseModel):
109
+ model_config = pdt.ConfigDict(strict=True, extra="forbid", frozen=True)
110
+
111
+ source: S3Uri
112
+ target: S3Uri
113
+
114
+ @property
115
+ def specificity(self) -> int:
116
+ return len(self.source.key.rstrip("/"))
117
+
118
+ def apply(self, uri: S3Uri) -> S3Uri | None:
119
+ if (uri.protocol, uri.bucket) != (self.source.protocol, self.source.bucket):
120
+ return None
121
+
122
+ source = self.source.key.rstrip("/")
123
+ target = self.target.key.rstrip("/")
124
+
125
+ if not source:
126
+ key = target + "/" + uri.key if target and uri.key else target or uri.key
127
+ elif uri.key == source:
128
+ key = self.target.key
129
+ elif uri.key.startswith(source + "/"):
130
+ suffix = uri.key[len(source):]
131
+ key = target + suffix if target else suffix[1:]
132
+ else:
133
+ return None
134
+
135
+ return S3Uri(protocol=self.target.protocol, bucket=self.target.bucket, key=key)
136
+
137
+
138
+ class S3UriMappingConfig(pdt.BaseModel):
139
+ model_config = pdt.ConfigDict(strict=True, extra="forbid")
140
+
141
+ mappings: list[S3UriMapping] = pdt.Field(default_factory=list)
142
+
143
+
144
+ class S3UriReinterpreter:
145
+ def __init__(self, config: S3UriMappingConfig, *, strict: bool = False):
146
+
147
+ sources = [(mapping.source.protocol, mapping.source.bucket, mapping.source.key.rstrip("/"))
148
+ for mapping in config.mappings]
149
+
150
+ if len(sources) != len(set(sources)):
151
+ raise ValueError("duplicate S3 mapping source")
152
+
153
+ self.mappings = sorted(config.mappings, key=lambda m: m.specificity, reverse=True)
154
+ self.strict = strict
155
+
156
+ @classmethod
157
+ def from_config(cls, config: JsonObject | S3UriMappingConfig, strict: bool = False) -> Self:
158
+ """Create from an already loaded and selected configuration."""
159
+ return cls(S3UriMappingConfig.model_validate(config), strict=strict)
160
+
161
+ def reinterpret(self, uri: str | os.PathLike[str] | S3UriLike) -> S3Uri:
162
+ """Resolve a URI value or a path-like object that preserves URI syntax.
163
+
164
+ ``pathlib.Path`` normalizes ``s3://`` to ``s3:/``, so construct S3
165
+ locations with strings or ``S3Uri`` instead.
166
+ """
167
+ if isinstance(uri, S3Uri):
168
+ parsed = uri
169
+ elif isinstance(uri, S3UriLike):
170
+ parsed = S3Uri.parse(uri.as_uri())
171
+ else:
172
+ parsed = S3Uri.parse(os.fspath(uri))
173
+
174
+ for mapping in self.mappings:
175
+ result = mapping.apply(parsed)
176
+ if result is not None:
177
+ return result
178
+
179
+ if self.strict:
180
+ raise ValueError(f"no mapping for '{uri!r}'")
181
+
182
+ return parsed
183
+
184
+
185
+ class S3ObjectMeta(pdt.BaseModel):
186
+ model_config = pdt.ConfigDict(strict=True, extra="forbid", frozen=True)
187
+
53
188
  key: str
54
189
  last_modified: datetime.datetime
55
190
  size: int
@@ -35,7 +35,6 @@ from plexus.pulse.common.utils.datautils import validate_dt_timezone, validate_s
35
35
  from plexus.pulse.common.utils.jsonutils import json_datetime_encoder
36
36
  from plexus.pulse.common.utils.ormutils import SQLiteDateTime, SequenceModelMixinProtocol
37
37
  from plexus.pulse.common.utils.ormutils import clone_sequence_model_instance, make_base_model, make_sequence_model_mixin
38
- from plexus.pulse.common.utils.sqlutils import escape_sql_like
39
38
 
40
39
  __all__ = [
41
40
  "versioned_identifier",
@@ -958,7 +957,7 @@ class TagCache(object):
958
957
  if tagset_version:
959
958
  query_stmt = query_stmt.filter(TagRecordTable.tagset_version == tagset_version)
960
959
  if tag_prefix:
961
- query_stmt = query_stmt.filter(TagRecordTable.tag.like(f"{escape_sql_like(tag_prefix)}%", escape="\\"))
960
+ query_stmt = query_stmt.filter(TagRecordTable.tag.startswith(tag_prefix, autoescape=True))
962
961
  if tagsets:
963
962
  if tagset_inverted:
964
963
  query_stmt = query_stmt.filter(
@@ -1033,7 +1032,7 @@ class TagCache(object):
1033
1032
  if tagset_version:
1034
1033
  query_stmt = query_stmt.filter(TagRecordTable.tagset_version == tagset_version)
1035
1034
  if tag_prefix:
1036
- query_stmt = query_stmt.filter(TagRecordTable.tag.like(f"{escape_sql_like(tag_prefix)}%", escape="\\"))
1035
+ query_stmt = query_stmt.filter(TagRecordTable.tag.startswith(tag_prefix, autoescape=True))
1037
1036
  if target_identifier:
1038
1037
  query_stmt = query_stmt.filter(TagTargetTable.identifier == target_identifier)
1039
1038
  if target_tagger_name:
@@ -1098,7 +1097,7 @@ class TagCache(object):
1098
1097
  if tagset_version:
1099
1098
  query_stmt = query_stmt.filter(TagRecordTable.tagset_version == tagset_version)
1100
1099
  if tag_prefix:
1101
- query_stmt = query_stmt.filter(TagRecordTable.tag.like(f"{escape_sql_like(tag_prefix)}%", escape="\\"))
1100
+ query_stmt = query_stmt.filter(TagRecordTable.tag.startswith(tag_prefix, autoescape=True))
1102
1101
  if tagsets:
1103
1102
  if tagset_inverted:
1104
1103
  query_stmt = query_stmt.filter(
@@ -1165,7 +1164,7 @@ class TagCache(object):
1165
1164
  if tagset_version:
1166
1165
  query_stmt = query_stmt.filter(TagRecordTable.tagset_version == tagset_version)
1167
1166
  if tag_prefix:
1168
- query_stmt = query_stmt.filter(TagRecordTable.tag.like(f"{escape_sql_like(tag_prefix)}%", escape="\\"))
1167
+ query_stmt = query_stmt.filter(TagRecordTable.tag.startswith(tag_prefix, autoescape=True))
1169
1168
  if tagsets:
1170
1169
  if tagset_inverted:
1171
1170
  query_stmt = query_stmt.filter(
@@ -1362,7 +1361,7 @@ class TagCache(object):
1362
1361
  if tagset_version:
1363
1362
  query_stmt = query_stmt.filter(TagRecordTable.tagset_version == tagset_version)
1364
1363
  if tag_prefix:
1365
- query_stmt = query_stmt.filter(TagRecordTable.tag.like(f"{escape_sql_like(tag_prefix)}%", escape="\\"))
1364
+ query_stmt = query_stmt.filter(TagRecordTable.tag.startswith(tag_prefix, autoescape=True))
1366
1365
  if tagsets:
1367
1366
  if tagset_inverted:
1368
1367
  query_stmt = query_stmt.filter(
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: plexus-python-pulse-common
3
- Version: 1.1.91
3
+ Version: 1.1.92
4
4
  Classifier: Programming Language :: Python :: 3
5
5
  Classifier: Programming Language :: Python :: 3.12
6
6
  Classifier: Programming Language :: Python :: 3.13
@@ -60,7 +60,6 @@ src/plexus/pulse/common/utils/jsonutils.py
60
60
  src/plexus/pulse/common/utils/ormutils.py
61
61
  src/plexus/pulse/common/utils/pathutils.py
62
62
  src/plexus/pulse/common/utils/s3utils.py
63
- src/plexus/pulse/common/utils/sqlutils.py
64
63
  src/plexus/pulse/common/utils/strutils.py
65
64
  src/plexus/pulse/common/utils/tagutils.py
66
65
  src/plexus/pulse/common/utils/testutils.py
@@ -1,11 +1,13 @@
1
1
  import contextlib
2
2
  import itertools
3
+ import pathlib
3
4
  import tempfile
4
5
  import unittest
5
6
 
6
7
  import ddt
7
8
  import moto
8
9
  import moto.server
10
+ import pydantic as pdt
9
11
  from plexus.common.utils.pathutils import make_path, scan_files
10
12
  from plexus.common.utils.randutils import randomizer
11
13
 
@@ -24,9 +26,164 @@ def moto_server_endpoint_url():
24
26
  server.stop()
25
27
 
26
28
 
29
+ class DummyPathLike:
30
+ def __init__(self, value: str):
31
+ self.value = value
32
+
33
+ def __fspath__(self):
34
+ return self.value
35
+
36
+
37
+ class DummyUriLike:
38
+ def __init__(self, value: str):
39
+ self.value = value
40
+
41
+ def as_uri(self):
42
+ return self.value
43
+
44
+
27
45
  @ddt.ddt
28
46
  class S3UtilsTest(unittest.TestCase):
29
47
 
48
+ @ddt.data(
49
+ ("s3://bucket", "s3", "bucket", ""),
50
+ ("s3://bucket/", "s3", "bucket", ""),
51
+ ("s3://bucket/a//b/", "s3", "bucket", "a//b/"),
52
+ ("s3://bucket/a%20b/../c", "s3", "bucket", "a%20b/../c"),
53
+ ("oss://bucket/key", "oss", "bucket", "key"),
54
+ ("cos://bucket/key", "cos", "bucket", "key"),
55
+ ("obs://bucket/key", "obs", "bucket", "key"),
56
+ )
57
+ @ddt.unpack
58
+ def test_s3_uri_parse(self, value, protocol, bucket, key):
59
+ uri = S3Uri.parse(value)
60
+ self.assertEqual((uri.protocol, uri.bucket, uri.key), (protocol, bucket, key))
61
+ self.assertIsInstance(uri, S3UriLike)
62
+ self.assertEqual(uri.as_uri(), f"{protocol}://{bucket}/{key}")
63
+ self.assertEqual(str(uri), uri.as_uri())
64
+ self.assertEqual(uri.model_dump(), uri.as_uri())
65
+ self.assertEqual(S3Uri.model_validate_json(uri.model_dump_json()), uri)
66
+
67
+ @ddt.data(
68
+ "",
69
+ "bucket/key",
70
+ "s3:/bucket/key",
71
+ "s3:///key",
72
+ "s3://user@bucket/key",
73
+ "s3://user:password@bucket/key",
74
+ "s3://bucket:9000/key",
75
+ "s3://bucket/key?version=1",
76
+ "s3://bucket/key#fragment",
77
+ 123,
78
+ None,
79
+ )
80
+ def test_s3_uri_parse__invalid(self, value):
81
+ with self.assertRaises(pdt.ValidationError):
82
+ S3Uri.parse(value)
83
+
84
+ @ddt.data(
85
+ ("s3://src", "s3://dst", "s3://src/file", "s3://dst/file"),
86
+ ("s3://src", "s3://dst/base", "s3://src/file", "s3://dst/base/file"),
87
+ ("s3://src", "s3://dst/base", "s3://src", "s3://dst/base"),
88
+ ("s3://src", "s3://dst/base", "s3://src//file/", "s3://dst/base//file/"),
89
+ ("s3://src/a", "oss://dst/b", "s3://src/a", "oss://dst/b"),
90
+ ("s3://src/a/", "s3://dst/b/", "s3://src/a/file/", "s3://dst/b/file/"),
91
+ ("s3://src/a", "s3://dst", "s3://src/a/file", "s3://dst/file"),
92
+ ("s3://src/a", "s3://dst/b", "s3://src/a//file", "s3://dst/b//file"),
93
+ ("s3://src/a", "s3://dst/b", "s3://src/abc", "s3://src/abc"),
94
+ ("s3://src/a", "s3://dst/b", "s3://other/a/file", "s3://other/a/file"),
95
+ ("s3://src/a", "s3://dst/b", "oss://src/a/file", "oss://src/a/file"),
96
+ )
97
+ @ddt.unpack
98
+ def test_s3_uri_reinterpret(self, source, target, value, expected):
99
+ interpreter = S3UriReinterpreter.from_config({"mappings": [{"source": source, "target": target}]})
100
+ result = interpreter.reinterpret(value)
101
+ self.assertIsInstance(result, S3Uri)
102
+ self.assertEqual(result.as_uri(), expected)
103
+
104
+ @ddt.data(False, True)
105
+ def test_s3_uri_reinterpret__longest_prefix(self, reverse):
106
+ rules = [
107
+ {"source": "s3://src", "target": "s3://default"},
108
+ {"source": "s3://src/a", "target": "s3://middle"},
109
+ {"source": "s3://src/a/b", "target": "s3://specific"},
110
+ {"source": "s3://specific", "target": "s3://chained"},
111
+ ]
112
+ interpreter = S3UriReinterpreter.from_config({"mappings": rules[::-1] if reverse else rules})
113
+ self.assertEqual(interpreter.reinterpret("s3://src/a/b/file").as_uri(), "s3://specific/file")
114
+
115
+ @ddt.data(
116
+ "s3://src/a/file",
117
+ S3Uri.parse("s3://src/a/file"),
118
+ DummyPathLike("s3://src/a/file"),
119
+ DummyUriLike("s3://src/a/file"),
120
+ )
121
+ def test_s3_uri_reinterpret__input_types(self, path):
122
+ interpreter = S3UriReinterpreter.from_config({"mappings": [{"source": "s3://src/a", "target": "s3://dst/b"}]})
123
+ self.assertEqual(interpreter.reinterpret(path).as_uri(), "s3://dst/b/file")
124
+
125
+ @ddt.data(False, True)
126
+ def test_s3_uri_reinterpret__unmatched(self, strict):
127
+ interpreter = S3UriReinterpreter.from_config({}, strict=strict)
128
+ value = S3Uri.parse("s3://src/file")
129
+ if strict:
130
+ with self.assertRaisesRegex(ValueError, "no mapping"):
131
+ interpreter.reinterpret(value)
132
+ else:
133
+ self.assertIs(interpreter.reinterpret(value), value)
134
+
135
+ @ddt.data(
136
+ pathlib.Path("s3://src/file"),
137
+ b"s3://src/file",
138
+ 123,
139
+ )
140
+ def test_s3_uri_reinterpret__invalid_input(self, value):
141
+ interpreter = S3UriReinterpreter.from_config({})
142
+ with self.assertRaises((ValueError, TypeError)):
143
+ interpreter.reinterpret(value)
144
+
145
+ @ddt.data(
146
+ {"mapping": []},
147
+ {"mappings": None},
148
+ {"mappings": ()},
149
+ {"mappings": [{"source": "s3://src"}]},
150
+ {"mappings": [{"source": 123, "target": "s3://dst"}]},
151
+ {"mappings": [{"source": "invalid", "target": "s3://dst"}]},
152
+ {"mappings": [{"source": "s3://src", "target": "s3://dst", "typo": True}]},
153
+ )
154
+ def test_s3_uri_reinterpret__invalid_config(self, config):
155
+ with self.assertRaises(pdt.ValidationError):
156
+ S3UriReinterpreter.from_config(config)
157
+
158
+ @ddt.data("s3://src/a", "s3://src/a/")
159
+ def test_s3_uri_reinterpret__duplicate_sources(self, duplicate):
160
+ with self.assertRaisesRegex(ValueError, "duplicate"):
161
+ S3UriReinterpreter.from_config(
162
+ {
163
+ "mappings": [
164
+ {"source": "s3://src/a", "target": "s3://dst/first"},
165
+ {"source": duplicate, "target": "s3://dst/second"},
166
+ ],
167
+ }
168
+ )
169
+
170
+ @ddt.data(False, True)
171
+ def test_s3_uri_reinterpret__config_round_trip(self, use_model):
172
+ raw = {
173
+ "mappings": [
174
+ {"source": "s3://src/a", "target": "s3://dst/first"},
175
+ {"source": "oss://src/a", "target": "oss://dst/second"},
176
+ ]
177
+ }
178
+
179
+ config = S3UriMappingConfig.model_validate(raw)
180
+ self.assertEqual(config.model_dump(), raw)
181
+ self.assertEqual(S3UriMappingConfig.model_validate_json(config.model_dump_json()), config)
182
+
183
+ interpreter = S3UriReinterpreter.from_config(config if use_model else raw)
184
+ self.assertEqual(interpreter.reinterpret("s3://src/a/file").as_uri(), "s3://dst/first/file")
185
+ self.assertEqual(interpreter.reinterpret("oss://src/a/file").as_uri(), "oss://dst/second/file")
186
+
30
187
  def test_s3_list_object__random_text_files(self):
31
188
  with moto.mock_aws(), s3_make_client() as client:
32
189
  client.create_bucket(Bucket="dummy-bucket")
@@ -1,9 +0,0 @@
1
- __all__ = [
2
- "escape_sql_like",
3
- ]
4
-
5
-
6
- def escape_sql_like(s: str | None) -> str:
7
- if s is None:
8
- raise ValueError("input string cannot be None")
9
- return s.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")