python-basekit 0.0.11__py3-none-any.whl

basekit/limiter.py

@@ -0,0 +1,179 @@
+import asyncio
+import time
+from collections import deque
+from collections.abc import Awaitable, Callable, Coroutine
+from contextlib import nullcontext
+from functools import wraps
+from typing import Annotated, Any, Literal, Self, override
+
+from pydantic import BaseModel, Field, PrivateAttr
+
+
+class AsyncRateLimiter(BaseModel):
+    limit: Annotated[int, Field(gt=0)]
+    window: Annotated[float, Field(gt=0)] = 60.0
+
+    _timestamps: deque[float] = PrivateAttr()
+    _lock: asyncio.Lock = PrivateAttr()
+    _waiters: deque[asyncio.Future[None]] = PrivateAttr()
+    _wake_task: asyncio.Task[None] | None = PrivateAttr()
+
+    @override
+    def model_post_init(self, context) -> None:
+        self._timestamps = deque()
+        self._lock = asyncio.Lock()
+        self._waiters = deque()
+        self._wake_task = None
+
+    def _evict(self, now: float, /) -> None:
+        cutoff: float = now - self.window
+        dq: deque[float] = self._timestamps
+
+        while dq and dq[0] < cutoff:
+            dq.popleft()
+
+    def _drain_waiters_unlocked(self, now: float, /) -> None:
+        self._evict(now)
+
+        while self._waiters and len(self._timestamps) < self.limit:
+            waiter: asyncio.Future[None] = self._waiters.popleft()
+            if waiter.done():
+                continue
+            self._timestamps.append(now)
+            waiter.set_result(None)
+
+    def _reschedule_wake_unlocked(self, now: float, /) -> None:
+        self._drain_waiters_unlocked(now)
+
+        if not self._waiters:
+            if self._wake_task is not None and not self._wake_task.done():
+                self._wake_task.cancel()
+            self._wake_task = None
+            return
+
+        if self._wake_task is not None and not self._wake_task.done():
+            return
+
+        wait: float = max(self._timestamps[0] + self.window - now, 0.0)
+        self._wake_task = asyncio.create_task(coro=self._wake_after(wait))
+
+    async def _wake_after(self, wait: float, /) -> None:
+        try:
+            await asyncio.sleep(delay=wait)
+        except asyncio.CancelledError:
+            return
+
+        async with self._lock:
+            if self._wake_task is not asyncio.current_task():
+                return
+            self._wake_task = None
+            self._reschedule_wake_unlocked(time.monotonic())
+
+    async def _acquire(self) -> None:
+        async with self._lock:
+            now: float = time.monotonic()
+            self._evict(now)
+            if not self._waiters and len(self._timestamps) < self.limit:
+                self._timestamps.append(now)
+                return
+
+            waiter: asyncio.Future[None] = asyncio.get_running_loop().create_future()
+            self._waiters.append(waiter)
+            self._reschedule_wake_unlocked(now)
+
+        try:
+            await waiter
+        except asyncio.CancelledError:
+            async with self._lock:
+                try:
+                    self._waiters.remove(waiter)
+                except ValueError:
+                    pass
+                self._reschedule_wake_unlocked(time.monotonic())
+            raise
+
+    async def __aenter__(self) -> Self:
+        await self._acquire()
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb) -> None:
+        return
+
+    def __call__[T, **P](
+        self, func: Callable[P, Awaitable[T]], /
+    ) -> Callable[P, Coroutine[Any, Any, T]]:
+        @wraps(wrapped=func)
+        async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
+            async with self:
+                return await func(*args, **kwargs)
+
+        return wrapper
+
+
+class AsyncConcurrencyLimiter(BaseModel):
+    limit: Annotated[int, Field(gt=0)]
+
+    _semaphore: asyncio.Semaphore = PrivateAttr()
+
+    @override
+    def model_post_init(self, context) -> None:
+        self._semaphore = asyncio.Semaphore(value=self.limit)
+
+    async def __aenter__(self) -> Self:
+        await self._semaphore.__aenter__()
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb) -> None:
+        await self._semaphore.__aexit__(exc_type=exc_type, exc=exc, tb=tb)
+
+    def __call__[T, **P](
+        self, func: Callable[P, Awaitable[T]], /
+    ) -> Callable[P, Coroutine[Any, Any, T]]:
+        @wraps(wrapped=func)
+        async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
+            async with self:
+                return await func(*args, **kwargs)
+
+        return wrapper
+
+
+class AsyncLimiter(BaseModel):
+    concurrency_limit: Annotated[int | None, Field(gt=0)]
+    rate_limit: Annotated[int | None, Field(gt=0)]
+    rate_window: Annotated[float, Field(gt=0)] = 60.0
+
+    _concurrency_limiter: AsyncConcurrencyLimiter | nullcontext = PrivateAttr()
+    _rate_limiter: AsyncRateLimiter | nullcontext = PrivateAttr()
+
+    @override
+    def model_post_init(self, context) -> None:
+        self._concurrency_limiter = (
+            AsyncConcurrencyLimiter(limit=self.concurrency_limit)
+            if self.concurrency_limit is not None
+            else nullcontext()
+        )
+        self._rate_limiter = (
+            AsyncRateLimiter(limit=self.rate_limit, window=self.rate_window)
+            if self.rate_limit is not None
+            else nullcontext()
+        )
+
+    async def __aenter__(self) -> Self:
+        await self._concurrency_limiter.__aenter__()
+        await self._rate_limiter.__aenter__()
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb) -> Literal[False]:
+        await self._concurrency_limiter.__aexit__(exc_type, exc, tb)
+        await self._rate_limiter.__aexit__(exc_type, exc, tb)
+        return False
+
+    def __call__[T, **P](
+        self, func: Callable[P, Awaitable[T]], /
+    ) -> Callable[P, Coroutine[Any, Any, T]]:
+        @wraps(wrapped=func)
+        async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
+            async with self:
+                return await func(*args, **kwargs)
+
+        return wrapper

basekit/utils/batch.py

@@ -0,0 +1,21 @@
+import asyncio
+from collections.abc import Awaitable
+
+import logfire
+
+
+async def run_batch[T](coros: list[Awaitable[T]], /, *, strict: bool) -> list[T]:
+    if strict:
+        return await asyncio.gather(*coros)
+
+    results: list[T | BaseException] = await asyncio.gather(
+        *coros, return_exceptions=True
+    )
+    instances: list[T] = [r for r in results if not isinstance(r, BaseException)]
+    errors: list[BaseException] = [r for r in results if isinstance(r, BaseException)]
+    if errors:
+        exception: BaseExceptionGroup[BaseException] = BaseExceptionGroup(
+            f"批量运行协程失败，{len(instances)}成功，{len(errors)}失败", errors
+        )
+        logfire.exception(str(object=exception), _exc_info=exception)
+    return instances

basekit/utils/console.py

@@ -0,0 +1,26 @@
+from pydantic import BaseModel
+from rich.console import Console
+from rich.table import Table, box
+
+
+def print_table[T: BaseModel](
+    console: Console, instances: list[T], /, *, title: str | None = None
+) -> None:
+    table = Table(title=title, box=box.SQUARE_DOUBLE_HEAD, show_lines=True)
+
+    if not instances:
+        table.add_column(header="无数据", justify="center", overflow="fold")
+        table.add_row("无数据")
+        console.print(table)
+        return
+
+    cls: type[T] = instances[0].__class__
+    for field in cls.model_fields.values():
+        table.add_column(header=field.description or "-", overflow="fold")
+    for instance in instances:
+        fields: list[str] = [
+            str(object=getattr(instance, field) or "-")
+            for field in cls.model_fields.keys()
+        ]
+        table.add_row(*fields)
+    console.print(table)

basekit/utils/html.py

@@ -0,0 +1,54 @@
+from urllib.parse import urljoin, urlparse
+
+from bs4 import BeautifulSoup, Tag
+from bs4.element import AttributeValueList
+
+from basekit.utils.misc import require_non_null, require_truthy
+
+
+def join_url(url: str, base: str, /) -> str:
+    if urlparse(url=url).scheme:
+        return url
+    if url.startswith("//"):
+        return f"{urlparse(url=base).scheme}:{url}"
+    return urljoin(base=base, url=url)
+
+
+def extract_attr(element: Tag, key: str, /) -> str | None:
+    value: str | AttributeValueList | None = element.get(key=key)
+    if value is None:
+        return None
+    if isinstance(value, str):
+        return value
+    return " ".join(value)
+
+
+def has_class(element: Tag, class_name: str, /) -> bool:
+    classes: str | None = extract_attr(element, "class")
+    if classes is None:
+        return False
+    return class_name in classes.split(sep=" ")
+
+
+def select_one(element: BeautifulSoup, selector: str, /) -> Tag:
+    result: Tag | None = element.select_one(selector=selector)
+    return require_non_null(result)
+
+
+def select_many(element: BeautifulSoup, selector: str, /) -> list[Tag]:
+    results: list[Tag] = element.select(selector=selector)
+    return require_truthy(results)
+
+
+def combine_htmls(*htmls: str) -> str:
+    contents: list[str] = []
+    for html in htmls:
+        soup = BeautifulSoup(markup=html, features="lxml")
+        for element in require_non_null(soup.body).children:
+            contents.append(str(object=element))
+    html: str = "".join(contents)
+    return BeautifulSoup(markup=html, features="lxml").decode()
+
+
+def extract_mime(content_type: str, /) -> str:
+    return content_type.split(sep=";", maxsplit=1)[0].strip().lower()

basekit/utils/jinja.py

@@ -0,0 +1,20 @@
+import re
+
+from jinja2 import Environment
+
+
+def split_sentences(text: str) -> list[str]:
+    if not text:
+        return []
+    return re.findall(pattern=r"[^。]+。?", string=text)
+
+
+MARKDOWN_ENV: Environment = Environment(
+    autoescape=False, keep_trailing_newline=True, lstrip_blocks=True, trim_blocks=True
+)
+MARKDOWN_ENV.policies["json.dumps_kwargs"]["ensure_ascii"] = False
+MARKDOWN_ENV.filters["split_sentences"] = split_sentences
+
+
+def render_template(template: str, /, **kwargs) -> str:
+    return MARKDOWN_ENV.from_string(source=template).render(**kwargs)

basekit/utils/markdown.py

@@ -0,0 +1,38 @@
+from flowmark import reformat_text
+from flowmark.linewrapping.markdown_filling import ListSpacing
+from markdown_it import MarkdownIt
+from markdown_it.tree import SyntaxTreeNode
+from markdownify import markdownify
+
+PARSER: MarkdownIt = (
+    MarkdownIt(config="commonmark")
+    .enable(names=["linkify", "replacements", "smartquotes"])
+    .enable(names=["table", "strikethrough"])
+)
+
+
+def format_markdown(markdown: str, /) -> str:
+    return reformat_text(
+        text=markdown, width=65535, semantic=False, list_spacing=ListSpacing.tight
+    )
+
+
+def convert_markdown(html: str, /) -> str:
+    markdown: str = markdownify(
+        html=html, heading_style="ATX", bullets="-", wrap_width=65535
+    )
+    return format_markdown(markdown)
+
+
+def parse_markdown(markdown: str, /) -> list[tuple[str, str]]:
+    lines: list[str] = markdown.splitlines(keepends=True)
+    root = SyntaxTreeNode(tokens=PARSER.parse(src=markdown))
+    blocks: list[tuple[str, str]] = []
+    for node in root.children:
+        if node.map is None:
+            continue
+        content: str = "".join(lines[node.map[0] : node.map[1]]).strip()
+        if not content:
+            continue
+        blocks.append((content, node.type))
+    return blocks

basekit/utils/mime.py

@@ -0,0 +1,39 @@
+from mimetypes import guess_extension, guess_type
+
+from filetype import Type
+from filetype import guess as guess_filetype
+
+DEFAULT_MIME = "application/octet-stream"
+DEFAULT_SUFFIX = ".bin"
+
+
+def guess_mime(
+    *, data: bytes | None = None, mime: str | None = None, suffix: str | None = None
+) -> str:
+    if data:
+        file_type: Type | None = guess_filetype(obj=data)
+        if file_type:
+            return file_type.mime
+    if mime:
+        return mime
+    if suffix:
+        suffix_mime: str | None = guess_type(url=f"test{suffix}")[0]
+        if suffix_mime:
+            return suffix_mime
+    return DEFAULT_MIME
+
+
+def guess_suffix(
+    *, data: bytes | None = None, mime: str | None = None, suffix: str | None = None
+) -> str:
+    if data:
+        file_type: Type | None = guess_filetype(obj=data)
+        if file_type:
+            return f".{file_type.extension}"
+    if mime:
+        mime_suffix: str | None = guess_extension(type=mime)
+        if mime_suffix:
+            return mime_suffix
+    if suffix:
+        return suffix
+    return DEFAULT_SUFFIX

basekit/utils/misc.py

@@ -0,0 +1,35 @@
+from collections.abc import Mapping
+from typing import Any
+
+
+def require_non_null[T](v: T | None, /) -> T:
+    if v is None:
+        raise ValueError(f"期望非空值: {v}")
+    return v
+
+
+def require_truthy[T](v: T | None, /) -> T:
+    if not v:
+        raise ValueError(f"期望真值: {v}")
+    return v
+
+
+def remove_space(s: str, /) -> str:
+    return " ".join(s.split())
+
+
+def recursive_merge(*args: Mapping | None) -> dict:
+    result: dict = {}
+
+    for mapping in args:
+        if mapping is None:
+            continue
+
+        for key, value in mapping.items():
+            current: Any | None = result.get(key)
+            if isinstance(current, Mapping) and isinstance(value, Mapping):
+                result[key] = recursive_merge(current, value)
+            else:
+                result[key] = value
+
+    return result

python_basekit-0.0.11.dist-info/METADATA

@@ -0,0 +1,46 @@
+Metadata-Version: 2.3
+Name: python-basekit
+Version: 0.0.11
+Summary: Python Base Kit
+Author: YanSH
+Author-email: YanSH <yansh97@foxmail.com>
+Requires-Dist: aiosqlite>=0.22.1
+Requires-Dist: beautifulsoup4>=4.14.3
+Requires-Dist: curl-cffi>=0.14.0
+Requires-Dist: filetype>=1.2.0
+Requires-Dist: flowmark>=0.6.5
+Requires-Dist: greenlet>=3.3.2
+Requires-Dist: httpx[http2]>=0.28.1
+Requires-Dist: jinja2>=3.1.6
+Requires-Dist: logfire>=4.27.0
+Requires-Dist: lxml>=6.0.2
+Requires-Dist: markdown-it-py>=4.0.0
+Requires-Dist: markdownify>=1.2.2
+Requires-Dist: pydantic>=2.12.5
+Requires-Dist: pydantic-settings>=2.13.1
+Requires-Dist: python-dotenv>=1.2.2
+Requires-Dist: rich>=14.3.3
+Requires-Dist: sqlalchemy>=2.0.48
+Requires-Dist: tenacity>=9.1.4
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+
+# Base Kit
+
+一个面向内部使用的 Python 基础工具包。
+
+## 环境
+
+Python `>=3.12`
+
+## 安装
+
+```bash
+uv add python-basekit
+```
+
+## 使用
+
+```python
+import basekit
+```

python_basekit-0.0.11.dist-info/RECORD

@@ -0,0 +1,27 @@
+basekit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+basekit/ai/clients/anthropic.py,sha256=iozRCRs3VC-JfZCRyiVPEIJ2vW-X4-IW0IcaCt5d5es,7620
+basekit/ai/clients/dashscope.py,sha256=QD_lSmSJYGkRtOOjg-C5CBHh3rN7tWf_ySHdnPWQ0js,4869
+basekit/ai/clients/gemini.py,sha256=MjM6JWzt7V4_zB3XR3_JtDRBzerUuDUjkRrsctiLTuQ,10795
+basekit/ai/clients/openai.py,sha256=pDypO4DaGygZVLupoBFYFfC_9qbA5rLVVs4GFK1_oVY,15255
+basekit/ai/schema.py,sha256=dTy3qff6YOUGBJN-fhhLjFEdcSGl_Agt35l1pb25JgU,5103
+basekit/ai/utils.py,sha256=P1VtktxlGkQn0suqS4xU5McG1zBHt8Ft75h19Mpv0II,1926
+basekit/cache/clients/sqlite.py,sha256=MpLInsD-xdE7-sIwAIJwhVrtMc2ASsTgthndo9mPumk,2981
+basekit/cache/schema.py,sha256=VYsE6nE5zARp0EYDaF2eqmVPh0qJkGkkP3-hGwq-P4A,1641
+basekit/cache/utils.py,sha256=hNQ7uqIadkMe6e6469UZXzCDYrPb0n52xwThEvFUQFk,3758
+basekit/database.py,sha256=LRt7rJGlg-le_SHKEu7WfnGjPd32yb5GyB_WswoIErE,1003
+basekit/http/clients/curl_cffi.py,sha256=G4FyvVVP3TR_K_6UZ4mGJaM--IYuBqzL0y1lCWRbH6M,3549
+basekit/http/clients/httpx.py,sha256=NgWlBmx164Kjmh07rk86gzZI4BhFn2tr_dh6VN9Li28,3501
+basekit/http/schema.py,sha256=P_QwrHWjsUEXmqx-rZXibrutA83HuOsVeH5OJfpB2bs,3061
+basekit/http/utils.py,sha256=SRLexTifocJ98Utxb1YSHoM5LBFa2r9sbCSeCDHJcvQ,941
+basekit/limiter.py,sha256=18CuaAppqcZVQUg7jHjC_ZnN-9Vx_0vB0VMqI4J2ekA,5885
+basekit/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+basekit/utils/batch.py,sha256=HNTc-12e3AkPC5dMUH2FbSinba2ZPxg9suPQeJd3utM,774
+basekit/utils/console.py,sha256=LPx6y0qlCd927z6FNqs6_rdva9QIFOMDrw4OcrY4rDE,867
+basekit/utils/html.py,sha256=RYg1oTe1J54eVNeiNiy8oMJZo-r4Tnz9yjdxU3Q4YKw,1676
+basekit/utils/jinja.py,sha256=8eoKz4dF4iSyvi8yInvtpq8_UYYul2jBVQGM1SxxJk0,568
+basekit/utils/markdown.py,sha256=lyDuvPhDxNUQzGWi2uP-F7qnaje9oizITH7FbZFikCo,1221
+basekit/utils/mime.py,sha256=nOMeAtDkVQP1rPGM5P-kkP0GeLhWIdhKMgs4paNcr6I,1070
+basekit/utils/misc.py,sha256=c5tkhPvyBpoHsvH4X7uuuit98AYEgAWwGU2EZyB4AGw,833
+python_basekit-0.0.11.dist-info/WHEEL,sha256=AKeAJsuKQDVE1yEMr94YEC7FtwPuFZZw2DmbXPmgtaE,81
+python_basekit-0.0.11.dist-info/METADATA,sha256=tkrxndeuUG_XmAyw8PjfL9PCy72rqWK-5xcZ8IU73XE,979
+python_basekit-0.0.11.dist-info/RECORD,,

python_basekit-0.0.11.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.11.10
+Root-Is-Purelib: true
+Tag: py3-none-any