queues-sdk 0.1.0__tar.gz

queues_sdk-0.1.0/.gitignore

@@ -0,0 +1,14 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+build/
+dist/
+.venv/
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.coverage
+htmlcov/
+.idea/
+.vscode/

queues_sdk-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 queues-sdk contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

queues_sdk-0.1.0/PKG-INFO

@@ -0,0 +1,105 @@
+Metadata-Version: 2.5
+Name: queues-sdk
+Version: 0.1.0
+Summary: Zero-loss async queue SDK: direct waiter handoff, backpressure waiting, pressure-gated spill to disk with FIFO drain. No network, no dependencies.
+Project-URL: Homepage, https://github.com/scapking/queues-sdk
+Project-URL: Source, https://github.com/scapking/queues-sdk
+Project-URL: Issues, https://github.com/scapking/queues-sdk/issues
+Author: queues-sdk contributors
+License: MIT
+License-File: LICENSE
+Keywords: asyncio,backpressure,pressure,queue,sdk,spill,zero-loss
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.10
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=7.4; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# queues-sdk
+
+**Zero-loss async queue SDK for Python.** Direct waiter handoff, backpressure waiting, pressure-gated spill to disk with FIFO drain. No network, no dependencies, stdlib only.
+
+Import name is `queues_sdk`; the PyPI distribution is `queues-sdk`.
+
+## Why
+
+Plain `asyncio.Queue` either blocks forever when full or drops messages if you cap it. Real pipelines need a third behaviour: **never lose, wait when possible, spill to disk when memory is tight, and soak in RAM when disk is tight.** queues-sdk implements exactly that on top of a lock-free deque + future handoff design.
+
+## Install
+
+```bash
+pip install queues-sdk
+```
+
+## Quickstart
+
+```python
+import asyncio
+from queues_sdk import Queues
+
+q = Queues()                     # maxsize=1024*512, thresholds built-in
+
+async def main():
+    await q.publish({"event": "signup"})           # unnamed: memory path
+    message = await q.consume()                    # backpressure-aware receive
+
+    await q.publish({"event": "audit"}, name="audit")   # named: pressure-gated
+    message = await q.consume(name="audit")             # auto-drains disk spill
+
+asyncio.run(main())
+```
+
+## Zero-loss semantics
+
+| Condition | Behaviour | Lost messages |
+|---|---|---|
+| normal watermarks, queue not full | lock-free deque append / future handoff | none |
+| queue full | publish **awaits space** (backpressure); queue-head never evicted | none |
+| memory soft line (>= memory_max) | named messages spill to disk as `batch-*.jsonl`, FIFO drain on consume | none |
+| disk hard line (>= disk_max) | named messages soak in RAM, spill resumes when disk recovers | none |
+| memory AND disk exhausted | no retreat | only here |
+
+## API
+
+- `Queues(maxsize=1024*512, memory_max=99, disk_max=99, spill_path="~/.local/queues_sdk/spill/", spill_batch=1024, watermarks=None)`
+- `await publish(message, name=None) -> None` — unnamed skips the pressure gate entirely (hot path)
+- `await consume(name=None) -> Any` — auto-drains the named spill buffer before waiting
+- `await drain(name, target_list, limit=None) -> int` — bounded FIFO refill from disk
+- `await discard(name) -> int` — wipe a spill buffer, returns dropped-message count
+- `state() -> "normal" | "soft" | "hard"` — current pressure mode
+- Five pre-built `asyncio.Queue` log lanes (`almightydb_service_logger_warning/info/debug/error/store`, each 1024*4) ship inside every instance
+
+Watermarks are injected: pass `watermarks=lambda: (memory_pct, disk_pct)`; the default provider reads `/proc/meminfo` + `statvfs("/")` on Linux and reports `(0, 0)` elsewhere.
+
+## Performance
+
+Measured on CPython 3.14 free-threaded, Linux, single process:
+
+| Path | Throughput |
+|---|---|
+| publish (unnamed, normal) | ~820k ops/s |
+| publish+consume pair (end-to-end TPS) | ~72k pairs/s |
+| named spill (soft line) | ~140k ops/s |
+| drain refill | ~356k ops/s |
+
+Spill writes are batched (1024 lines per file): expect one ~1-4 ms disk pause per batch under the soft line.
+
+## Docs
+
+- [Getting started](docs/01-getting-started.md)
+- [API reference](docs/02-api-reference.md)
+- [Performance & latency profile](docs/03-performance.md)
+
+## License
+
+MIT

queues_sdk-0.1.0/README.md

@@ -0,0 +1,78 @@
+# queues-sdk
+
+**Zero-loss async queue SDK for Python.** Direct waiter handoff, backpressure waiting, pressure-gated spill to disk with FIFO drain. No network, no dependencies, stdlib only.
+
+Import name is `queues_sdk`; the PyPI distribution is `queues-sdk`.
+
+## Why
+
+Plain `asyncio.Queue` either blocks forever when full or drops messages if you cap it. Real pipelines need a third behaviour: **never lose, wait when possible, spill to disk when memory is tight, and soak in RAM when disk is tight.** queues-sdk implements exactly that on top of a lock-free deque + future handoff design.
+
+## Install
+
+```bash
+pip install queues-sdk
+```
+
+## Quickstart
+
+```python
+import asyncio
+from queues_sdk import Queues
+
+q = Queues()                     # maxsize=1024*512, thresholds built-in
+
+async def main():
+    await q.publish({"event": "signup"})           # unnamed: memory path
+    message = await q.consume()                    # backpressure-aware receive
+
+    await q.publish({"event": "audit"}, name="audit")   # named: pressure-gated
+    message = await q.consume(name="audit")             # auto-drains disk spill
+
+asyncio.run(main())
+```
+
+## Zero-loss semantics
+
+| Condition | Behaviour | Lost messages |
+|---|---|---|
+| normal watermarks, queue not full | lock-free deque append / future handoff | none |
+| queue full | publish **awaits space** (backpressure); queue-head never evicted | none |
+| memory soft line (>= memory_max) | named messages spill to disk as `batch-*.jsonl`, FIFO drain on consume | none |
+| disk hard line (>= disk_max) | named messages soak in RAM, spill resumes when disk recovers | none |
+| memory AND disk exhausted | no retreat | only here |
+
+## API
+
+- `Queues(maxsize=1024*512, memory_max=99, disk_max=99, spill_path="~/.local/queues_sdk/spill/", spill_batch=1024, watermarks=None)`
+- `await publish(message, name=None) -> None` — unnamed skips the pressure gate entirely (hot path)
+- `await consume(name=None) -> Any` — auto-drains the named spill buffer before waiting
+- `await drain(name, target_list, limit=None) -> int` — bounded FIFO refill from disk
+- `await discard(name) -> int` — wipe a spill buffer, returns dropped-message count
+- `state() -> "normal" | "soft" | "hard"` — current pressure mode
+- Five pre-built `asyncio.Queue` log lanes (`almightydb_service_logger_warning/info/debug/error/store`, each 1024*4) ship inside every instance
+
+Watermarks are injected: pass `watermarks=lambda: (memory_pct, disk_pct)`; the default provider reads `/proc/meminfo` + `statvfs("/")` on Linux and reports `(0, 0)` elsewhere.
+
+## Performance
+
+Measured on CPython 3.14 free-threaded, Linux, single process:
+
+| Path | Throughput |
+|---|---|
+| publish (unnamed, normal) | ~820k ops/s |
+| publish+consume pair (end-to-end TPS) | ~72k pairs/s |
+| named spill (soft line) | ~140k ops/s |
+| drain refill | ~356k ops/s |
+
+Spill writes are batched (1024 lines per file): expect one ~1-4 ms disk pause per batch under the soft line.
+
+## Docs
+
+- [Getting started](docs/01-getting-started.md)
+- [API reference](docs/02-api-reference.md)
+- [Performance & latency profile](docs/03-performance.md)
+
+## License
+
+MIT

queues_sdk-0.1.0/docs/01-getting-started.md

@@ -0,0 +1,57 @@
+# 01 — Getting started
+
+## Requirements
+
+- Python >= 3.10
+- No runtime dependencies.
+
+## Install
+
+```bash
+pip install queues-sdk
+```
+
+## Create a queue
+
+```python
+from queues_sdk import Queues
+
+q = Queues()
+```
+
+Constructor arguments (all optional, thresholds are **built into the instance**, never read from config files):
+
+| Argument | Default | Meaning |
+|---|---|---|
+| `maxsize` | `1024*512` | in-memory buffer capacity |
+| `memory_max` | `99` | memory soft line, 0-100 |
+| `disk_max` | `99` | disk hard line, 0-100 |
+| `spill_path` | `~/.local/queues_sdk/spill/` | spill root directory |
+| `spill_batch` | `1024` | lines per spilled batch file |
+| `watermarks` | `None` | provider returning `(memory_pct, disk_pct)` |
+
+## Publish and consume
+
+```python
+await q.publish(message)                  # unnamed: memory path
+await q.publish(message, name="audit")    # named: pressure-gated spill
+message = await q.consume()               # memory path first, then waits
+message = await q.consume(name="audit")   # auto-drains the spill buffer first
+```
+
+- Unnamed calls skip the pressure gate entirely — that is the hot path.
+- Named calls route through the gate: `spill` when a watermark is crossed, `pass` otherwise.
+- `consume` blocks with a future handoff; producers deliver messages directly without touching the buffer.
+
+## Watermark injection
+
+```python
+q = Queues(watermarks=lambda: (75, 40))   # memory 75%, disk 40%
+```
+
+The default provider computes both percentages from `/proc/meminfo` (`MemTotal`/`MemAvailable`) and `statvfs("/")`; on non-Linux platforms it reports `(0, 0)`.
+
+## Next
+
+- [API reference](02-api-reference.md)
+- [Performance & latency profile](03-performance.md)

queues_sdk-0.1.0/docs/02-api-reference.md

@@ -0,0 +1,41 @@
+# 02 — API reference
+
+## `Queues`
+
+```python
+Queues(maxsize=1024*512, memory_max=99, disk_max=99, spill_path="~/.local/queues_sdk/spill/", spill_batch=1024, watermarks=None) -> Queues
+```
+
+## `async publish(message, name=None) -> None`
+
+- `name=None`: memory fast path; if the buffer is full the producer awaits space (backpressure, zero loss, queue head never evicted).
+- `name=str`: goes through the pressure gate. On the soft line the message is buffered and spilled to disk in batches; on the hard line it soaks in RAM until the disk recovers.
+- The five service log lanes are for the host application to use directly; `publish` does not touch them.
+
+## `async consume(name=None) -> Any`
+
+1. Buffer non-empty: pop the head, wake one space-waiting producer if there is room.
+2. `name` given and spill residue exists: bounded `drain` into the buffer, then pop.
+3. Otherwise: attach a waiter future and suspend; `publish` hands the message over directly.
+
+Cancellation while waiting removes the waiter and re-raises `CancelledError` — a cancelled consume never silently returns a fake `None` mixed with real data.
+
+## `async drain(name, target, limit=None) -> int`
+
+Bounded FIFO refill from disk into `target` (a list is typical). Spilled files are read oldest-first, each deleted after it is fully consumed (at-least-once on crash). Corrupt lines quarantine the whole file as `.bad` instead of blocking the stream.
+
+## `async discard(name) -> int`
+
+Wipes `pending`, `draining` and `checked` state plus the on-disk directory; the return value is the total number of messages discarded (pending lines + spill file lines) for reconciliation.
+
+## `state() -> str`
+
+`"normal"` / `"soft"` / `"hard"` from the injected watermarks; any provider failure degrades to `"normal"`.
+
+## Exceptions
+
+- `QueuesError` — explicit API failure surfaced to callers.
+
+## Error logging
+
+Internal failures go through `__log_error` into the `queues_sdk` standard-library logger at `ERROR` level with a structured dict (id, UTC time, method, type, value, file/line, pid/thread, traceback). Nothing is swallowed silently.

queues_sdk-0.1.0/docs/03-performance.md

@@ -0,0 +1,33 @@
+# 03 — Performance & latency profile
+
+Measured on CPython 3.14 free-threaded, Linux, single process, in-process alternation against the upstream engine implementation.
+
+## Throughput
+
+| Path | Throughput |
+|---|---|
+| `publish`, unnamed, normal watermarks | ~820k ops/s |
+| `publish`+`consume` pair (end-to-end TPS) | ~72k pairs/s |
+| named spill on soft line | ~140k ops/s |
+| `drain` refill | ~356k ops/s |
+
+Design points behind these numbers:
+
+- publish without a waiter: lock-free `deque.append`, no locks, no event-loop round trips.
+- publish with a waiter: `future.set_result` handoff — the message never touches the buffer.
+- unnamed publishes bypass the pressure gate coroutine entirely.
+- refill reads a whole spilled file at once and deserialises in one pass.
+
+## Latency profile: steady vs paused
+
+The hot path is **steady-state**: no timers, no periodic flushes, allocations are short-lived objects reclaimed by reference counting, so there is no GC-shaped pause built into the design.
+
+The only **episodic** pause source is batched spill: measured median `publish` latency on the soft line is ~10 µs, with a ~1-4 ms max spike on the batch boundary (one synchronous `open`+`write` per 1024 lines). Under the hard line there are no writes at all — messages soak in RAM.
+
+Backpressure (`queue full`) is event-driven suspension, not polling and not a pause: producers resume exactly when a consumer frees one slot.
+
+## Tuning
+
+- `spill_batch` — lower it to shorten the per-file write spike; raise it to amortise fewer, bigger writes.
+- `maxsize` — raise it to absorb bursts fully in RAM.
+- Watermark thresholds (`memory_max`/`disk_max`) — lower them to enter the safe paths earlier; they are constructor arguments, tuned per deployment, never state.

queues_sdk-0.1.0/examples/backpressure.py

@@ -0,0 +1,23 @@
+import asyncio
+
+from queues_sdk import Queues
+
+
+async def main():
+    q = Queues(maxsize=8)                        # small buffer on purpose
+
+    async def producer(count):
+        for i in range(count):
+            await q.publish({"i": i})            # blocks when full, never drops
+
+    async def consumer(count):
+        for i in range(count):
+            print("consumed:", (await q.consume())["i"])
+
+    await asyncio.gather(producer(10), consumer(5))
+    await asyncio.sleep(0.05)
+    await consumer(5)                            # drains the rest
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

queues_sdk-0.1.0/examples/pressure_spill.py

@@ -0,0 +1,25 @@
+import asyncio
+
+from queues_sdk import Queues
+
+
+async def main():
+    holder = {"memory": 0, "disk": 0}
+    q = Queues(maxsize=1024*4, watermarks=lambda: (int(holder["memory"]), int(holder["disk"])))
+
+    holder["memory"] = 100                       # cross the memory soft line
+    for i in range(5):
+        await q.publish({"m": i}, name="audit")  # buffered into pending
+    print("soaked in pending:", sum(len(v) for v in q.pending.values()))
+
+    holder["memory"] = 0                         # pressure recovers
+    drained = []
+    pulled = await q.drain("audit", drained)
+    print("drained:", pulled, "in FIFO order:", [x["m"] for x in drained])
+
+    dropped = await q.discard("audit")
+    print("dropped after drain:", dropped)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

queues_sdk-0.1.0/examples/quickstart.py

@@ -0,0 +1,21 @@
+import asyncio
+
+from queues_sdk import Queues
+
+
+async def main():
+    q = Queues()
+
+    await q.publish({"event": "signup", "user": 1})
+    message = await q.consume()
+    print("unnamed round-trip:", message)
+
+    await q.publish({"event": "audit", "user": 2}, name="audit")
+    message = await q.consume(name="audit")
+    print("named round-trip:", message)
+
+    print("pressure state:", q.state())
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

queues_sdk-0.1.0/pyproject.toml

@@ -0,0 +1,48 @@
+[build-system]
+requires = ["hatchling>=1.18"]
+build-backend = "hatchling.build"
+
+[project]
+name = "queues-sdk"
+version = "0.1.0"
+description = "Zero-loss async queue SDK: direct waiter handoff, backpressure waiting, pressure-gated spill to disk with FIFO drain. No network, no dependencies."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+authors = [{ name = "queues-sdk contributors" }]
+keywords = ["queue", "asyncio", "backpressure", "zero-loss", "spill", "pressure", "sdk"]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: System :: Distributed Computing",
+  "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = []
+
+[project.optional-dependencies]
+dev = ["pytest>=7.4", "pytest-asyncio>=0.23"]
+
+[project.urls]
+Homepage = "https://github.com/scapking/queues-sdk"
+Source = "https://github.com/scapking/queues-sdk"
+Issues = "https://github.com/scapking/queues-sdk/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/queues_sdk"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src", "tests", "docs", "examples", "README.md", "LICENSE"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-q"
+asyncio_mode = "auto"
+
+[tool.ruff]
+line-length = 110

queues_sdk-0.1.0/src/queues_sdk/__init__.py

@@ -0,0 +1,28 @@
+"""queues_sdk - zero-loss async queue SDK.
+
+One class does the work::
+
+    from queues_sdk import Queues
+
+    q = Queues()                       # maxsize=1024*512, thresholds built-in
+    await q.publish(message)           # unnamed: memory path, never dropped
+    await q.publish(message, name="audit")   # named: pressure-gated spill
+    message = await q.consume()              # backpressure-aware receive
+    message = await q.consume(name="audit")  # auto-drains disk spill first
+
+Pressure semantics (memory/disk watermarks, 0-100):
+
+- disk hard line  -> named messages soak in RAM, spill resumes when disk recovers
+- memory soft line -> named messages spill to disk as batch-*.jsonl, FIFO drain
+- queue full      -> publish awaits space instead of raising; queue-head never evicted
+
+Zero-loss promise: messages are only lost if memory AND disk are exhausted.
+Thresholds are constructor arguments, never config files. Import name is
+``queues_sdk``; the PyPI distribution is ``queues-sdk``.
+"""
+
+from ._version import __version__
+from .core import Queues
+from .errors import QueuesError
+
+__all__ = ["Queues", "QueuesError", "__version__"]

queues_sdk-0.1.0/src/queues_sdk/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

queues_sdk-0.1.0/src/queues_sdk/core.py

@@ -0,0 +1,224 @@
+import sys, os, time, asyncio, uuid, traceback, threading, collections, json, shutil, datetime, pathlib
+from typing import Callable
+
+from queues_sdk.watermarks import default_watermarks
+from queues_sdk.errors import QueuesError
+
+
+
+
+
+
+class Queues:
+    def __init__(self, maxsize: int = 1024*512, memory_max: int = 99, disk_max: int = 99, spill_path: str = "~/.local/queues_sdk/spill/", spill_batch: int = 1024, watermarks: Callable[[], tuple[int, int]] | None = None):
+        self.pending = dict()
+        self.draining = dict()
+        self.checked = set()
+        self.pressure_memory_max = int(memory_max)
+        self.pressure_disk_max = int(disk_max)
+        self.pressure_spill_path = str(spill_path)
+        self.pressure_spill_batch = int(spill_batch)
+        self._watermarks = watermarks if watermarks is not None else default_watermarks
+        self.db = collections.deque(maxlen=int(maxsize))
+        self.db_append = self.db.append
+        self.db_popleft = self.db.popleft
+        self.event = asyncio.Event()
+        self.waiters = collections.deque()
+        self.waiters_append = self.waiters.append
+        self.waiters_popleft = self.waiters.popleft
+        self.space_waiters = collections.deque()
+        self.space_waiters_append = self.space_waiters.append
+        self.space_waiters_popleft = self.space_waiters.popleft
+        self.almightydb_service_logger_warning = asyncio.Queue(maxsize=1024*4)
+        self.almightydb_service_logger_info    = asyncio.Queue(maxsize=1024*4)
+        self.almightydb_service_logger_debug   = asyncio.Queue(maxsize=1024*4)
+        self.almightydb_service_logger_error   = asyncio.Queue(maxsize=1024*4)
+        self.almightydb_service_logger_store   = asyncio.Queue(maxsize=1024*4)
+
+
+
+
+
+    def state(self) -> str:
+        try:
+            # 磁盘过硬线 → hard；内存过硬线 → soft；否则 normal。
+            memory, disk = self._watermarks()
+            if int(disk) >= int(self.pressure_disk_max): return "hard"
+            if int(memory) >= int(self.pressure_memory_max): return "soft"
+            return "normal"
+
+        except BaseException: return "normal"
+
+
+
+
+
+    async def gate(self, name: str | None, message) -> str:
+        try:
+            # 写侧统一入口：'spill' 压力缓冲（硬线滞留内存不写盘/软线溢出落盘）/ 'pass' 内存路径。
+            memory, disk = self._watermarks()
+            hard = int(disk) >= int(self.pressure_disk_max)
+            if name is None: return "pass"
+            if name not in self.checked:
+                self.checked.add(name)
+                directory = self._spill_directory(name)
+                if directory.is_dir() and any(directory.glob("*.jsonl")): self.draining[name] = True
+            if hard or int(memory) >= int(self.pressure_memory_max) or self.draining.get(name):
+                lines = self.pending.setdefault(name, list())
+                lines.append(json.dumps(message, ensure_ascii=False, default=str))
+                # 攒批到界且磁盘未硬线时落盘一个 batch-*.jsonl，行缓冲随即清空。
+                if len(lines) >= int(self.pressure_spill_batch) and not hard:
+                    directory = self._spill_directory(name)
+                    directory.mkdir(parents=True, exist_ok=True)
+                    path = directory / "batch-{}.jsonl".format(time.monotonic_ns())
+                    with open(path, "w", encoding="utf-8") as handle: handle.write("\n".join(lines) + "\n")
+                    lines.clear()
+                return "spill"
+            return "pass"
+
+        except BaseException as error: await self.__log_error("gate", error); return "pass"
+
+
+
+
+
+    def _spill_directory(self, name: str):
+        return pathlib.Path(self.pressure_spill_path).expanduser() / str(name)
+
+
+
+
+
+    async def drain(self, name: str, target, limit: int | None = None) -> int:
+        try:
+            # 有界回灌：盘上旧 batch 文件在前 FIFO，pending 就地落成最新文件在后，顺序自然正确。
+            limit = int(self.pressure_spill_batch) if limit is None else int(limit)
+            pulled = int(0); lines = self.pending.get(name); directory = self._spill_directory(name)
+            if lines:
+                directory.mkdir(parents=True, exist_ok=True)
+                path = directory / "batch-{}.jsonl".format(time.monotonic_ns())
+                with open(path, "w", encoding="utf-8") as handle: handle.write("\n".join(lines) + "\n")
+                lines.clear()
+            if directory.is_dir():
+                for path in sorted(directory.glob("*.jsonl")):
+                    if pulled >= limit: break
+                    try:
+                        with open(path, encoding="utf-8") as handle: batch = [line.strip() for line in handle if line.strip()]
+                        target.extend(json.loads(line) for line in batch); pulled = pulled + len(batch)
+                        path.unlink()
+                    except BaseException:
+                        # 毒文件改名 .bad 隔离，不阻塞 FIFO。
+                        os.replace(path, directory / (path.name + ".bad"))
+            self.checked.add(name)
+            if self.pending.get(name) or (directory.is_dir() and any(directory.glob("*.jsonl"))): self.draining[name] = True
+            else: self.draining.pop(name, None)
+            return pulled
+
+        except BaseException as error: await self.__log_error("drain", error); return int(0)
+
+
+
+
+
+    async def discard(self, name: str) -> int:
+        try:
+            # 清盘并整盘对账：丢弃条数 = pending 行数 + 盘上 batch 行数。
+            dropped = len(self.pending.pop(name, list()))
+            self.draining.pop(name, None)
+            self.checked.discard(name)
+            directory = self._spill_directory(name)
+            if directory.is_dir():
+                for path in directory.glob("*.jsonl"):
+                    with open(path, encoding="utf-8") as handle: dropped = dropped + len([line for line in handle if line.strip()])
+            shutil.rmtree(directory, ignore_errors=True)
+            return int(dropped)
+
+        except BaseException as error: await self.__log_error("discard", error); return int(0)
+
+
+
+
+
+    async def publish(self, message, name: str | None = None):
+        try:
+            # 未命名消息直走内存路径（零依赖快路径）；命名消息过压力门。
+            action = "pass" if name is None else await self.gate(name, message)
+            if action == "spill": return None
+            if self.waiters and not self.waiters[0].done(): self.waiters_popleft().set_result(message); return None
+            if self.waiters: self.waiters_popleft()
+            # 容量满 → 背压等待消费腾位，不拒绝不挤队首（零丢失）。
+            while self.db.maxlen is not None and len(self.db) >= self.db.maxlen:
+                producer = asyncio.get_running_loop().create_future()
+                self.space_waiters_append(producer)
+                try:
+                    await producer
+                except asyncio.CancelledError:
+                    if producer in self.space_waiters: self.space_waiters.remove(producer)
+                    raise
+            self.db_append(message)
+
+        except BaseException as error:
+            # 取消必须上抛：吸收会让 producer 误以为消息已入队（丢失语义）。
+            if isinstance(error, asyncio.CancelledError): raise
+            await self.__log_error("publish", error); return None; return None
+
+
+
+
+
+    async def consume(self, name: str | None = None):
+        try:
+            if len(self.db) > 0:
+                message = self.db_popleft()
+                if self.space_waiters and self.db.maxlen is not None and len(self.db) < self.db.maxlen: self.space_waiters_popleft().set_result(None)
+                return message
+            if name and (self.draining.get(name) or self.pending.get(name)): await self.drain(name, self.db)
+            if len(self.db) > 0: return self.db_popleft()
+            waiter = asyncio.get_running_loop().create_future()
+            self.waiters_append(waiter)
+            # 取消时清理等待者并上抛（吸收取消会让消费端拿到 None 混淆空值语义）。
+            try:
+                message = await waiter
+            except asyncio.CancelledError:
+                if waiter in self.waiters: self.waiters.remove(waiter)
+                raise
+            return message
+
+        except BaseException as error:
+            # 取消必须上抛：吸收会让消费循环把 None 当消息、teardown 永久等待。
+            if isinstance(error, asyncio.CancelledError): raise
+            await self.__log_error("consume", error); return None; return None
+
+
+
+
+
+    async def main(self):
+        try:
+            pass
+        except BaseException as error: await self.__log_error("main", error)
+
+
+
+
+
+    async def __log_error(self, method: str, error: BaseException):
+        try:
+            import logging
+            frames = traceback.extract_tb(error.__traceback__)
+            frame = frames[-1] if frames else None
+            detail = {
+                "error_id": uuid.uuid4().hex,
+                "error_time": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+                "error_method": "{}.{}".format(type(self).__name__, method),
+                "error_type": type(error).__name__,
+                "error_value": str(error),
+                "error_file": os.path.abspath(frame.filename) if frame else "unknown",
+                "error_line": frame.lineno if frame else 0,
+                "error_pid": os.getpid(),
+                "error_thread": threading.current_thread().name,
+                "error_traceback": "".join(traceback.format_exception(type(error), error, error.__traceback__)),
+            }
+            logging.getLogger("queues_sdk").error("[queues_sdk error] [{}]".format(detail))
+
+        except BaseException: pass

queues_sdk-0.1.0/src/queues_sdk/errors.py

@@ -0,0 +1,2 @@
+class QueuesError(RuntimeError):
+    """queues-sdk 业务异常：对外 API 的显式失败（与内部守卫背压语义区分）。"""

queues_sdk-0.1.0/src/queues_sdk/watermarks.py

@@ -0,0 +1,19 @@
+import os
+
+
+
+
+
+def default_watermarks() -> tuple[int, int]:
+    try:
+        memory_total = int(0); memory_available = int(0)
+        with open("/proc/meminfo", encoding="utf-8") as handle:
+            for line in handle:
+                if line.startswith("MemTotal:"): memory_total = int(line.split()[1])
+                elif line.startswith("MemAvailable:"): memory_available = int(line.split()[1])
+        usage = os.statvfs("/")
+        disk_percent = int(100 - usage.f_bavail * 100 / max(int(usage.f_blocks), 1))
+        memory_percent = int(100 - memory_available * 100 / max(memory_total, 1))
+        return (int(memory_percent), int(disk_percent))
+
+    except OSError: return (int(0), int(0))

queues_sdk-0.1.0/tests/conftest.py

@@ -0,0 +1,21 @@
+import pytest
+
+from queues_sdk import Queues
+from queues_sdk.watermarks import default_watermarks
+
+
+
+
+
+@pytest.fixture
+def q() -> Queues:
+    return Queues(maxsize=1024*4, watermarks=lambda: (int(0), int(0)))
+
+
+
+
+
+@pytest.fixture
+def watermarks():
+    holder = {"memory": int(0), "disk": int(0)}
+    return holder, lambda: (int(holder["memory"]), int(holder["disk"]))

queues_sdk-0.1.0/tests/test_core.py

@@ -0,0 +1,68 @@
+import asyncio
+
+import pytest
+
+from queues_sdk import Queues, QueuesError
+
+
+
+
+
+async def test_fifo_order(q: Queues):
+    for i in range(6): await q.publish({"i": i})
+    assert [await q.consume() for _ in range(6)] == [{"i": i} for i in range(6)]
+
+
+
+
+
+async def test_waiter_handoff(q: Queues):
+    seen = []
+    async def consumer():
+        while True: seen.append(await q.consume())
+    task = asyncio.create_task(consumer())
+    await asyncio.sleep(0.01)
+    for i in range(6):
+        await q.publish({"i": i})
+        await asyncio.sleep(0.01)
+    assert [item["i"] for item in seen] == [0, 1, 2, 3, 4, 5]
+    task.cancel()
+    await asyncio.sleep(0)
+
+
+
+
+
+async def test_full_backpressure_zero_loss(q: Queues):
+    capacity = int(q.db.maxlen)
+    for i in range(capacity): await q.publish({"i": i})
+    tasks = [asyncio.create_task(q.publish({"i": capacity + j})) for j in range(3)]
+    await asyncio.sleep(0.05)
+    assert all(not task.done() for task in tasks)
+    for _ in range(3): await q.consume()
+    await asyncio.sleep(0.05)
+    assert all(task.done() for task in tasks)
+    assert len(q.db) == capacity
+    assert int(q.db[0]["i"]) == 3 and int(q.db[-1]["i"]) == capacity + 2
+    q.db.clear()
+
+
+
+
+
+async def test_consume_cancel_cleanup(q: Queues):
+    task = asyncio.create_task(q.consume())
+    await asyncio.sleep(0.01)
+    task.cancel()
+    with pytest.raises(asyncio.CancelledError): await task
+    assert len(q.waiters) == 0
+    await q.publish({"i": 1})
+    assert (await q.consume())["i"] == 1
+
+
+
+
+
+async def test_named_error_type():
+    error = QueuesError("probe")
+    assert str(error) == "probe"

queues_sdk-0.1.0/tests/test_pressure.py

@@ -0,0 +1,70 @@
+import asyncio
+
+from queues_sdk import Queues
+from queues_sdk.watermarks import default_watermarks
+
+
+
+
+
+async def test_soft_spill_and_drain_fifo():
+    holder = {"memory": int(100), "disk": int(0)}
+    q = Queues(maxsize=1024*4, spill_batch=1024, watermarks=lambda: (int(holder["memory"]), int(holder["disk"])))
+    for i in range(3): await q.publish({"m": i}, name="probe")
+    assert sum(len(v) for v in q.pending.values()) == 3
+    holder["memory"] = 0
+    drained = []
+    pulled = await q.drain("probe", drained)
+    assert int(pulled) == 3 and drained == [{"m": 0}, {"m": 1}, {"m": 2}]
+    dropped = await q.discard("probe")
+    assert int(dropped) == 0 and not q.pending.get("probe")
+
+
+
+
+
+async def test_hard_soaks_in_ram_without_disk_write():
+    holder = {"memory": int(0), "disk": int(100)}
+    q = Queues(maxsize=1024*4, spill_batch=1024, watermarks=lambda: (int(holder["memory"]), int(holder["disk"])))
+    for i in range(1103): await q.publish({"m": i}, name="probe")
+    assert sum(len(v) for v in q.pending.values()) == 1103
+    import pathlib
+    directory = pathlib.Path(q.pressure_spill_path).expanduser() / "probe"
+    assert not (directory.is_dir() and any(directory.glob("*.jsonl")))
+    holder["disk"] = 0
+    drained = []
+    pulled = await q.drain("probe", drained)
+    assert int(pulled) == 1103 and [x["m"] for x in drained][:3] == [0, 1, 2]
+    await q.discard("probe")
+
+
+
+
+
+async def test_discard_reports_dropped():
+    holder = {"memory": int(100), "disk": int(0)}
+    q = Queues(maxsize=1024*4, watermarks=lambda: (int(holder["memory"]), int(holder["disk"])))
+    for i in range(3): await q.publish({"m": i}, name="probe")
+    dropped = await q.discard("probe")
+    assert int(dropped) == 3
+
+
+
+
+
+async def test_unnamed_bypasses_gate(q: Queues):
+    for i in range(6): await q.publish({"i": i})
+    assert [item["i"] for item in q.db] == [0, 1, 2, 3, 4, 5]
+
+
+
+
+
+async def test_state_three_modes():
+    holder = {"memory": int(0), "disk": int(0)}
+    q = Queues(watermarks=lambda: (int(holder["memory"]), int(holder["disk"])))
+    assert q.state() == "normal"
+    holder["memory"] = 100
+    assert q.state() == "soft"
+    holder["memory"] = 0; holder["disk"] = 100
+    assert q.state() == "hard"

queues_sdk-0.1.0/tests/test_watermarks.py

@@ -0,0 +1,29 @@
+import queues_sdk
+
+from queues_sdk import Queues
+from queues_sdk.watermarks import default_watermarks
+
+
+
+
+
+def test_version_exported():
+    assert queues_sdk.__version__ == "0.1.0"
+
+
+
+
+
+def test_default_watermarks_shape():
+    memory, disk = default_watermarks()
+    assert 0 <= int(memory) <= 100 and 0 <= int(disk) <= 100
+
+
+
+
+
+def test_custom_thresholds():
+    q = Queues(memory_max=97, disk_max=98, spill_batch=512, maxsize=1024)
+    assert int(q.pressure_memory_max) == 97 and int(q.pressure_disk_max) == 98
+    assert int(q.pressure_spill_batch) == 512 and int(q.db.maxlen) == 1024
+    assert int(q.almightydb_service_logger_store.maxsize) == 1024*4