dycw-utilities 0.130.2__py3-none-any.whl → 0.131.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dycw-utilities
3
- Version: 0.130.2
3
+ Version: 0.131.1
4
4
  Author-email: Derek Wan <d.wan@icloud.com>
5
5
  License-File: LICENSE
6
6
  Requires-Python: >=3.12
@@ -1,4 +1,4 @@
1
- utilities/__init__.py,sha256=xiyBegs4PBVyJ6bA-ugSHN76oMsqk3uRB1M8N8U1_NQ,60
1
+ utilities/__init__.py,sha256=7cvfTXJSLM1ajMZOJ6keAbIOlGbRAUMdkqIQoooZ6Bk,60
2
2
  utilities/aiolimiter.py,sha256=mD0wEiqMgwpty4XTbawFpnkkmJS6R4JRsVXFUaoitSU,628
3
3
  utilities/altair.py,sha256=Gpja-flOo-Db0PIPJLJsgzAlXWoKUjPU1qY-DQ829ek,9156
4
4
  utilities/asyncio.py,sha256=lvdgBhuMtxq0dpiwF9g2WMMrit3kqXibN1V5NZ4xdbo,38046
@@ -24,7 +24,7 @@ utilities/getpass.py,sha256=DfN5UgMAtFCqS3dSfFHUfqIMZX2shXvwphOz_6J6f6A,103
24
24
  utilities/git.py,sha256=oi7-_l5e9haSANSCvQw25ufYGoNahuUPHAZ6114s3JQ,1191
25
25
  utilities/hashlib.py,sha256=SVTgtguur0P4elppvzOBbLEjVM3Pea0eWB61yg2ilxo,309
26
26
  utilities/http.py,sha256=WcahTcKYRtZ04WXQoWt5EGCgFPcyHD3EJdlMfxvDt-0,946
27
- utilities/hypothesis.py,sha256=UnUMJmeqwJuK7uyUqw_i3opUYzVKud4RMG0RMOSRBQY,44463
27
+ utilities/hypothesis.py,sha256=y6a5Ilokrlvu7hNpPxUFWKy5p7vra1Oe6yEj1g-1Fng,42747
28
28
  utilities/importlib.py,sha256=mV1xT_O_zt_GnZZ36tl3xOmMaN_3jErDWY54fX39F6Y,429
29
29
  utilities/inflect.py,sha256=DbqB5Q9FbRGJ1NbvEiZBirRMxCxgrz91zy5jCO9ZIs0,347
30
30
  utilities/ipython.py,sha256=V2oMYHvEKvlNBzxDXdLvKi48oUq2SclRg5xasjaXStw,763
@@ -50,7 +50,7 @@ utilities/pickle.py,sha256=Bhvd7cZl-zQKQDFjUerqGuSKlHvnW1K2QXeU5UZibtg,657
50
50
  utilities/platform.py,sha256=48IOKx1IC6ZJXWG-b56ZQptITcNFhWRjELW72o2dGTA,2398
51
51
  utilities/polars.py,sha256=QlmUpYTqHNkcLnWOQh1TW22W2QyLzvifCvBcbsqhpdE,63272
52
52
  utilities/polars_ols.py,sha256=Uc9V5kvlWZ5cU93lKZ-cfAKdVFFw81tqwLW9PxtUvMs,5618
53
- utilities/pottery.py,sha256=luBJV4LXQv6MaQdJmTN5R1QOz98visTuf3ayWFT8bHY,1538
53
+ utilities/pottery.py,sha256=-YikdIQFqhFXifyMq5R5z6V6N8y639gAnVVXR4jXxdc,3568
54
54
  utilities/pqdm.py,sha256=foRytQybmOQ05pjt5LF7ANyzrIa--4ScDE3T2wd31a4,3118
55
55
  utilities/psutil.py,sha256=RtbLKOoIJhqrJmEoHDBVeSD-KPzshtS0FtRXBP9_w2s,3751
56
56
  utilities/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -70,7 +70,7 @@ utilities/shelve.py,sha256=HZsMwK4tcIfg3sh0gApx4-yjQnrY4o3V3ZRimvRhoW0,738
70
70
  utilities/slack_sdk.py,sha256=ltmzv68aa73CJGqTDvt8L9vDm22YU9iOCo3NCiNd3vA,4347
71
71
  utilities/socket.py,sha256=K77vfREvzoVTrpYKo6MZakol0EYu2q1sWJnnZqL0So0,118
72
72
  utilities/sqlalchemy.py,sha256=IuQ7CIVNl29TG6i81K6fam8NmTmPjtA6OiIN4nIM9W8,37935
73
- utilities/sqlalchemy_polars.py,sha256=OPrB_Aqh8KE3hfNqvvXSzqVH5CYgIYrDH13WoLdCzbw,15510
73
+ utilities/sqlalchemy_polars.py,sha256=hApbjQUY-XgKfAXcun8gDP2lGh5LxrudnCpbG_hrYa0,14968
74
74
  utilities/statsmodels.py,sha256=koyiBHvpMcSiBfh99wFUfSggLNx7cuAw3rwyfAhoKpQ,3410
75
75
  utilities/streamlit.py,sha256=U9PJBaKP1IdSykKhPZhIzSPTZsmLsnwbEPZWzNhJPKk,2955
76
76
  utilities/string.py,sha256=XmU-s04qIV_tODnKl2pQiwmHaxzgOqRKU-RyzdrfvSE,620
@@ -90,7 +90,7 @@ utilities/warnings.py,sha256=un1LvHv70PU-LLv8RxPVmugTzDJkkGXRMZTE2-fTQHw,1771
90
90
  utilities/whenever.py,sha256=QbXgFAPuUL7PCp2hajmIP-FFIfIR1J6Y0TxJbeoj60I,18434
91
91
  utilities/zipfile.py,sha256=24lQc9ATcJxHXBPc_tBDiJk48pWyRrlxO2fIsFxU0A8,699
92
92
  utilities/zoneinfo.py,sha256=-5j7IQ9nb7gR43rdgA7ms05im-XuqhAk9EJnQBXxCoQ,1874
93
- dycw_utilities-0.130.2.dist-info/METADATA,sha256=iA0TZllOrUsCc1Q7--u4Z6r_Lyl7hj1tMYqHQGDCmLA,12989
94
- dycw_utilities-0.130.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
95
- dycw_utilities-0.130.2.dist-info/licenses/LICENSE,sha256=gppZp16M6nSVpBbUBrNL6JuYfvKwZiKgV7XoKKsHzqo,1066
96
- dycw_utilities-0.130.2.dist-info/RECORD,,
93
+ dycw_utilities-0.131.1.dist-info/METADATA,sha256=wuztGM9vYp7gFU1LsLTuS2JxrKHKBH-AAMFyvjQ6Uo4,12989
94
+ dycw_utilities-0.131.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
95
+ dycw_utilities-0.131.1.dist-info/licenses/LICENSE,sha256=gppZp16M6nSVpBbUBrNL6JuYfvKwZiKgV7XoKKsHzqo,1066
96
+ dycw_utilities-0.131.1.dist-info/RECORD,,
utilities/__init__.py CHANGED
@@ -1,3 +1,3 @@
1
1
  from __future__ import annotations
2
2
 
3
- __version__ = "0.130.2"
3
+ __version__ = "0.131.1"
utilities/hypothesis.py CHANGED
@@ -96,10 +96,8 @@ if TYPE_CHECKING:
96
96
 
97
97
  from hypothesis.database import ExampleDatabase
98
98
  from numpy.random import RandomState
99
- from sqlalchemy.ext.asyncio import AsyncEngine
100
99
 
101
100
  from utilities.numpy import NDArrayB, NDArrayF, NDArrayI, NDArrayO
102
- from utilities.sqlalchemy import Dialect, TableOrORMInstOrClass
103
101
  from utilities.types import Duration, Number, RoundMode
104
102
 
105
103
 
@@ -1117,51 +1115,6 @@ def slices(
1117
1115
  ##
1118
1116
 
1119
1117
 
1120
- _STRATEGY_DIALECTS: list[Dialect] = ["sqlite", "postgresql"]
1121
- _SQLALCHEMY_ENGINE_DIALECTS = sampled_from(_STRATEGY_DIALECTS)
1122
-
1123
-
1124
- async def sqlalchemy_engines(
1125
- data: DataObject,
1126
- /,
1127
- *tables_or_orms: TableOrORMInstOrClass,
1128
- dialect: MaybeSearchStrategy[Dialect] = _SQLALCHEMY_ENGINE_DIALECTS,
1129
- ) -> AsyncEngine:
1130
- """Strategy for generating sqlalchemy engines."""
1131
- from utilities.sqlalchemy import create_async_engine
1132
-
1133
- dialect_: Dialect = draw2(data, dialect)
1134
- if "CI" in environ: # pragma: no cover
1135
- _ = assume(dialect_ == "sqlite")
1136
- match dialect_:
1137
- case "sqlite":
1138
- temp_path = data.draw(temp_paths())
1139
- path = Path(temp_path, "db.sqlite")
1140
- engine = create_async_engine("sqlite+aiosqlite", database=str(path))
1141
-
1142
- class EngineWithPath(type(engine)): ...
1143
-
1144
- engine_with_path = EngineWithPath(engine.sync_engine)
1145
- cast(
1146
- "Any", engine_with_path
1147
- ).temp_path = temp_path # keep `temp_path` alive
1148
- return engine_with_path
1149
- case "postgresql": # skipif-ci-and-not-linux
1150
- from utilities.sqlalchemy import ensure_tables_dropped
1151
-
1152
- engine = create_async_engine(
1153
- "postgresql+asyncpg", host="localhost", port=5432, database="testing"
1154
- )
1155
- with assume_does_not_raise(ConnectionRefusedError):
1156
- await ensure_tables_dropped(engine, *tables_or_orms)
1157
- return engine
1158
- case _: # pragma: no cover
1159
- raise NotImplementedError(dialect)
1160
-
1161
-
1162
- ##
1163
-
1164
-
1165
1118
  @composite
1166
1119
  def str_arrays(
1167
1120
  draw: DrawFn,
@@ -1512,7 +1465,6 @@ __all__ = [
1512
1465
  "sets_fixed_length",
1513
1466
  "setup_hypothesis_profiles",
1514
1467
  "slices",
1515
- "sqlalchemy_engines",
1516
1468
  "str_arrays",
1517
1469
  "temp_dirs",
1518
1470
  "temp_paths",
utilities/pottery.py CHANGED
@@ -1,50 +1,116 @@
1
1
  from __future__ import annotations
2
2
 
3
- import asyncio
4
3
  from contextlib import asynccontextmanager, suppress
5
- from typing import TYPE_CHECKING
4
+ from dataclasses import dataclass
5
+ from typing import TYPE_CHECKING, override
6
6
 
7
7
  from pottery import AIORedlock
8
8
  from pottery.exceptions import ReleaseUnlockedLock
9
9
  from redis.asyncio import Redis
10
10
 
11
+ from utilities.asyncio import sleep_dur, timeout_dur
11
12
  from utilities.datetime import MILLISECOND, SECOND, datetime_duration_to_float
12
13
  from utilities.iterables import always_iterable
13
14
 
14
15
  if TYPE_CHECKING:
15
- from collections.abc import AsyncIterator
16
+ from collections.abc import AsyncIterator, Iterable
16
17
 
17
18
  from utilities.types import Duration, MaybeIterable
18
19
 
19
20
 
20
21
  @asynccontextmanager
21
- async def yield_locked_resource(
22
+ async def yield_access(
22
23
  redis: MaybeIterable[Redis],
23
24
  key: str,
24
25
  /,
25
26
  *,
26
- duration: Duration = 10 * SECOND,
27
+ num: int = 1,
28
+ timeout_acquire: Duration | None = None,
29
+ timeout_release: Duration = 10 * SECOND,
27
30
  sleep: Duration = MILLISECOND,
28
- ) -> AsyncIterator[AIORedlock]:
29
- """Yield a locked resource."""
31
+ throttle: Duration | None = None,
32
+ ) -> AsyncIterator[None]:
33
+ """Acquire access to a locked resource, amongst 1 of multiple connections."""
34
+ if num <= 0:
35
+ raise _YieldAccessNumLocksError(key=key, num=num)
30
36
  masters = ( # skipif-ci-and-not-linux
31
37
  {redis} if isinstance(redis, Redis) else set(always_iterable(redis))
32
38
  )
33
- duration_use = datetime_duration_to_float(duration) # skipif-ci-and-not-linux
34
- lock = AIORedlock( # skipif-ci-and-not-linux
35
- key=key,
36
- masters=masters,
37
- auto_release_time=duration_use,
38
- context_manager_timeout=duration_use,
39
+ auto_release_time = datetime_duration_to_float( # skipif-ci-and-not-linux
40
+ timeout_release
39
41
  )
40
- sleep_use = datetime_duration_to_float(sleep) # skipif-ci-and-not-linux
41
- while not await lock.acquire(): # pragma: no cover
42
- _ = await asyncio.sleep(sleep_use)
42
+ locks = [ # skipif-ci-and-not-linux
43
+ AIORedlock(
44
+ key=f"{key}_{i}_of_{num}",
45
+ masters=masters,
46
+ auto_release_time=auto_release_time,
47
+ )
48
+ for i in range(1, num + 1)
49
+ ]
50
+ lock: AIORedlock | None = None # skipif-ci-and-not-linux
43
51
  try: # skipif-ci-and-not-linux
44
- yield lock
52
+ lock = await _get_first_available_lock(
53
+ key, locks, num=num, timeout=timeout_acquire, sleep=sleep
54
+ )
55
+ yield
45
56
  finally: # skipif-ci-and-not-linux
46
- with suppress(ReleaseUnlockedLock):
47
- await lock.release()
57
+ await sleep_dur(duration=throttle)
58
+ if lock is not None:
59
+ with suppress(ReleaseUnlockedLock):
60
+ await lock.release()
48
61
 
49
62
 
50
- __all__ = ["yield_locked_resource"]
63
+ async def _get_first_available_lock(
64
+ key: str,
65
+ locks: Iterable[AIORedlock],
66
+ /,
67
+ *,
68
+ num: int = 1,
69
+ timeout: Duration | None = None,
70
+ sleep: Duration | None = None,
71
+ ) -> AIORedlock:
72
+ locks = list(locks) # skipif-ci-and-not-linux
73
+ error = _YieldAccessUnableToAcquireLockError( # skipif-ci-and-not-linux
74
+ key=key, num=num, timeout=timeout
75
+ )
76
+ async with timeout_dur( # skipif-ci-and-not-linux
77
+ duration=timeout, error=error
78
+ ):
79
+ while True:
80
+ if (result := await _get_first_available_lock_if_any(locks)) is not None:
81
+ return result
82
+ await sleep_dur(duration=sleep)
83
+
84
+
85
+ async def _get_first_available_lock_if_any(
86
+ locks: Iterable[AIORedlock], /
87
+ ) -> AIORedlock | None:
88
+ for lock in locks: # skipif-ci-and-not-linux
89
+ if await lock.acquire(blocking=False):
90
+ return lock
91
+ return None # skipif-ci-and-not-linux
92
+
93
+
94
+ @dataclass(kw_only=True, slots=True)
95
+ class YieldAccessError(Exception):
96
+ key: str
97
+ num: int
98
+
99
+
100
+ @dataclass(kw_only=True, slots=True)
101
+ class _YieldAccessNumLocksError(YieldAccessError):
102
+ @override
103
+ def __str__(self) -> str:
104
+ return f"Number of locks for {self.key!r} must be positive; got {self.num}"
105
+
106
+
107
+ @dataclass(kw_only=True, slots=True)
108
+ class _YieldAccessUnableToAcquireLockError(YieldAccessError):
109
+ timeout: Duration | None
110
+
111
+ @override
112
+ def __str__(self) -> str:
113
+ return f"Unable to acquire any 1 of {self.num} locks for {self.key!r} after {self.timeout}" # skipif-ci-and-not-linux
114
+
115
+
116
+ __all__ = ["YieldAccessError", "yield_access"]
@@ -25,7 +25,6 @@ from polars import (
25
25
  )
26
26
  from sqlalchemy import Column, Select, select
27
27
  from sqlalchemy.exc import DuplicateColumnError
28
- from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
29
28
 
30
29
  from utilities.asyncio import timeout_dur
31
30
  from utilities.functions import identity
@@ -61,6 +60,7 @@ if TYPE_CHECKING:
61
60
  )
62
61
 
63
62
  from polars._typing import PolarsDataType, SchemaDict
63
+ from sqlalchemy.ext.asyncio import AsyncEngine
64
64
  from sqlalchemy.sql import ColumnCollection
65
65
  from sqlalchemy.sql.base import ReadOnlyColumnCollection
66
66
  from tenacity.retry import RetryBaseT
@@ -307,22 +307,6 @@ async def select_to_dataframe(
307
307
  **kwargs: Any,
308
308
  ) -> DataFrame | Iterable[DataFrame] | AsyncIterable[DataFrame]:
309
309
  """Read a table from a database into a DataFrame."""
310
- if not issubclass(AsyncEngine, type(engine)):
311
- # for handling testing
312
- engine = create_async_engine(engine.url)
313
- return await select_to_dataframe(
314
- sel,
315
- engine,
316
- snake=snake,
317
- time_zone=time_zone,
318
- batch_size=batch_size,
319
- in_clauses=in_clauses,
320
- in_clauses_chunk_size=in_clauses_chunk_size,
321
- chunk_size_frac=chunk_size_frac,
322
- timeout=timeout,
323
- error=error,
324
- **kwargs,
325
- )
326
310
  if snake:
327
311
  sel = _select_to_dataframe_apply_snake(sel)
328
312
  schema = _select_to_dataframe_map_select_to_df_schema(sel, time_zone=time_zone)