iheap 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- iheap-0.1.0/Cargo.lock +137 -0
- iheap-0.1.0/Cargo.toml +15 -0
- iheap-0.1.0/LICENSE +21 -0
- iheap-0.1.0/PKG-INFO +242 -0
- iheap-0.1.0/README.md +219 -0
- iheap-0.1.0/crates/iheap-core/Cargo.toml +10 -0
- iheap-0.1.0/crates/iheap-core/examples/bench.rs +98 -0
- iheap-0.1.0/crates/iheap-core/src/group.rs +223 -0
- iheap-0.1.0/crates/iheap-core/src/heap.rs +589 -0
- iheap-0.1.0/crates/iheap-core/src/key.rs +97 -0
- iheap-0.1.0/crates/iheap-core/src/lib.rs +94 -0
- iheap-0.1.0/crates/iheap-core/tests/model.rs +120 -0
- iheap-0.1.0/crates/iheap-py/Cargo.toml +17 -0
- iheap-0.1.0/crates/iheap-py/src/lib.rs +155 -0
- iheap-0.1.0/pyproject.toml +45 -0
- iheap-0.1.0/python/iheap/__init__.py +15 -0
- iheap-0.1.0/python/iheap/_iheap.pyi +151 -0
- iheap-0.1.0/python/iheap/_pqdict.py +546 -0
- iheap-0.1.0/python/iheap/py.typed +0 -0
iheap-0.1.0/Cargo.lock
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# This file is automatically @generated by Cargo.
|
|
2
|
+
# It is not intended for manual editing.
|
|
3
|
+
version = 3
|
|
4
|
+
|
|
5
|
+
[[package]]
|
|
6
|
+
name = "heck"
|
|
7
|
+
version = "0.5.0"
|
|
8
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
9
|
+
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
|
10
|
+
|
|
11
|
+
[[package]]
|
|
12
|
+
name = "iheap-core"
|
|
13
|
+
version = "0.1.0"
|
|
14
|
+
|
|
15
|
+
[[package]]
|
|
16
|
+
name = "iheap-py"
|
|
17
|
+
version = "0.1.0"
|
|
18
|
+
dependencies = [
|
|
19
|
+
"iheap-core",
|
|
20
|
+
"pyo3",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
[[package]]
|
|
24
|
+
name = "libc"
|
|
25
|
+
version = "0.2.191"
|
|
26
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
27
|
+
checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
|
|
28
|
+
|
|
29
|
+
[[package]]
|
|
30
|
+
name = "once_cell"
|
|
31
|
+
version = "1.21.4"
|
|
32
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
33
|
+
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
|
34
|
+
|
|
35
|
+
[[package]]
|
|
36
|
+
name = "portable-atomic"
|
|
37
|
+
version = "1.15.0"
|
|
38
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
39
|
+
checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
|
|
40
|
+
|
|
41
|
+
[[package]]
|
|
42
|
+
name = "proc-macro2"
|
|
43
|
+
version = "1.0.107"
|
|
44
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
45
|
+
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
|
46
|
+
dependencies = [
|
|
47
|
+
"unicode-ident",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
[[package]]
|
|
51
|
+
name = "pyo3"
|
|
52
|
+
version = "0.30.1"
|
|
53
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
54
|
+
checksum = "229ca13b8a15bb0d1487b81ab975bfcd97513416499a8312011b6036e42e0de4"
|
|
55
|
+
dependencies = [
|
|
56
|
+
"libc",
|
|
57
|
+
"once_cell",
|
|
58
|
+
"portable-atomic",
|
|
59
|
+
"pyo3-build-config",
|
|
60
|
+
"pyo3-ffi",
|
|
61
|
+
"pyo3-macros",
|
|
62
|
+
]
|
|
63
|
+
|
|
64
|
+
[[package]]
|
|
65
|
+
name = "pyo3-build-config"
|
|
66
|
+
version = "0.30.1"
|
|
67
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
68
|
+
checksum = "70130e2b6dd1ee589a8788026a429561ca5c40fa7a2fa7914c2282eddae24700"
|
|
69
|
+
dependencies = [
|
|
70
|
+
"target-lexicon",
|
|
71
|
+
]
|
|
72
|
+
|
|
73
|
+
[[package]]
|
|
74
|
+
name = "pyo3-ffi"
|
|
75
|
+
version = "0.30.1"
|
|
76
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
77
|
+
checksum = "5e7ff73c555c286d212d4ce7e5fbda845a973c4f61bd5f160f5f2c3c2a04d1c4"
|
|
78
|
+
dependencies = [
|
|
79
|
+
"libc",
|
|
80
|
+
"pyo3-build-config",
|
|
81
|
+
]
|
|
82
|
+
|
|
83
|
+
[[package]]
|
|
84
|
+
name = "pyo3-macros"
|
|
85
|
+
version = "0.30.1"
|
|
86
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
87
|
+
checksum = "c46d2e5862c7bb8207e3b639f713dda3c499933444784154c11d013c5a691b60"
|
|
88
|
+
dependencies = [
|
|
89
|
+
"proc-macro2",
|
|
90
|
+
"pyo3-macros-backend",
|
|
91
|
+
"quote",
|
|
92
|
+
"syn",
|
|
93
|
+
]
|
|
94
|
+
|
|
95
|
+
[[package]]
|
|
96
|
+
name = "pyo3-macros-backend"
|
|
97
|
+
version = "0.30.1"
|
|
98
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
99
|
+
checksum = "38b89f066f2ae72a2aef52d1537ca2fcd104440ebfadee06e97aa90738a25b9b"
|
|
100
|
+
dependencies = [
|
|
101
|
+
"heck",
|
|
102
|
+
"proc-macro2",
|
|
103
|
+
"quote",
|
|
104
|
+
"syn",
|
|
105
|
+
]
|
|
106
|
+
|
|
107
|
+
[[package]]
|
|
108
|
+
name = "quote"
|
|
109
|
+
version = "1.0.47"
|
|
110
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
111
|
+
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
|
112
|
+
dependencies = [
|
|
113
|
+
"proc-macro2",
|
|
114
|
+
]
|
|
115
|
+
|
|
116
|
+
[[package]]
|
|
117
|
+
name = "syn"
|
|
118
|
+
version = "3.0.9"
|
|
119
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
120
|
+
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
|
|
121
|
+
dependencies = [
|
|
122
|
+
"proc-macro2",
|
|
123
|
+
"quote",
|
|
124
|
+
"unicode-ident",
|
|
125
|
+
]
|
|
126
|
+
|
|
127
|
+
[[package]]
|
|
128
|
+
name = "target-lexicon"
|
|
129
|
+
version = "0.13.5"
|
|
130
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
131
|
+
checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
|
132
|
+
|
|
133
|
+
[[package]]
|
|
134
|
+
name = "unicode-ident"
|
|
135
|
+
version = "1.0.27"
|
|
136
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
137
|
+
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
iheap-0.1.0/Cargo.toml
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
[workspace]
|
|
2
|
+
resolver = "2"
|
|
3
|
+
members = ["crates/iheap-core", "crates/iheap-py"]
|
|
4
|
+
|
|
5
|
+
[workspace.package]
|
|
6
|
+
version = "0.1.0"
|
|
7
|
+
edition = "2021"
|
|
8
|
+
license = "MIT"
|
|
9
|
+
repository = "https://github.com/nvbkdw/iheap"
|
|
10
|
+
rust-version = "1.88" # std::hint::select_unpredictable
|
|
11
|
+
|
|
12
|
+
[profile.release]
|
|
13
|
+
lto = "fat"
|
|
14
|
+
codegen-units = 1
|
|
15
|
+
panic = "abort"
|
iheap-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ryan Huang
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
iheap-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: iheap
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Classifier: Development Status :: 4 - Beta
|
|
5
|
+
Classifier: Intended Audience :: Developers
|
|
6
|
+
Classifier: Programming Language :: Python :: 3
|
|
7
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
8
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
9
|
+
Classifier: Programming Language :: Rust
|
|
10
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
11
|
+
Classifier: Typing :: Typed
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Summary: Fast indexed priority queue (pqdict-style) with a Rust core
|
|
14
|
+
Keywords: priority queue,heap,indexed heap,pqdict,dijkstra
|
|
15
|
+
Author-email: Ryan Huang <ryan.huang@noetik.ai>
|
|
16
|
+
License-Expression: MIT
|
|
17
|
+
Requires-Python: >=3.10
|
|
18
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
19
|
+
Project-URL: Documentation, https://nvbkdw.github.io/iheap/
|
|
20
|
+
Project-URL: Issues, https://github.com/nvbkdw/iheap/issues
|
|
21
|
+
Project-URL: Repository, https://github.com/nvbkdw/iheap
|
|
22
|
+
|
|
23
|
+
# iheap
|
|
24
|
+
|
|
25
|
+
A fast **indexed priority queue** for Python with a Rust core. It's a drop-in
|
|
26
|
+
style replacement for [`pqdict`](https://github.com/nvictus/pqdict), and is
|
|
27
|
+
**3–20× faster**.
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
from iheap import PQDict
|
|
31
|
+
|
|
32
|
+
pq = PQDict({"a": 3, "b": 1, "c": 2}) # key -> priority, min on top
|
|
33
|
+
pq["d"] = 0 # insert
|
|
34
|
+
pq["a"] = -1 # update priority in O(log n)
|
|
35
|
+
del pq["c"] # delete any key in O(log n)
|
|
36
|
+
pq.top() # 'a'
|
|
37
|
+
list(pq.popitems()) # [('a', -1), ('d', 0), ('b', 1)]
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
```
|
|
41
|
+
pip install iheap
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
📖 **Documentation:** <https://nvbkdw.github.io/iheap/>. It covers a
|
|
45
|
+
getting-started guide, the API reference, the benchmark and development notes.
|
|
46
|
+
|
|
47
|
+
Prebuilt `abi3` wheels cover CPython ≥ 3.10 on Linux (glibc and musl, x86_64
|
|
48
|
+
and aarch64), macOS (x86_64 and arm64) and Windows (x64).
|
|
49
|
+
|
|
50
|
+
## Design
|
|
51
|
+
|
|
52
|
+
Only priorities (as `f64`) and integer handles cross into Rust. Keys and
|
|
53
|
+
values (the payload) never leave Python.
|
|
54
|
+
|
|
55
|
+
```
|
|
56
|
+
Python (iheap.PQDict) Rust (iheap-core)
|
|
57
|
+
┌───────────────────────────────────┐ ┌─────────────────────────────────────┐
|
|
58
|
+
│ _handles: dict[key -> handle] │ │ 8-ary tournament tree │
|
|
59
|
+
│ _keys: list[handle -> key] │──▶│ leaves[h] = priority of handle h │
|
|
60
|
+
│ _values: list[handle -> value] │ │ node = winner of 8 children │
|
|
61
|
+
└───────────────────────────────────┘ └─────────────────────────────────────┘
|
|
62
|
+
only f64 priorities and u32 handles cross ▲
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
* **Handles, not keys.** The Rust core gives out dense `u32` handles and
|
|
66
|
+
recycles them, so Python keeps its key/value data in plain lists indexed by
|
|
67
|
+
handle. Rust never holds a Python object and never calls back into Python.
|
|
68
|
+
* **An 8-ary tournament (winner) tree, not a binary heap.** Leaf `h` holds the
|
|
69
|
+
priority of handle `h`, so no position map is needed. Each internal node
|
|
70
|
+
stores the winner of 8 children. A sibling group is one 64- or 128-byte
|
|
71
|
+
cache-line block.
|
|
72
|
+
* **The path is known in advance.** Every operation writes one leaf and
|
|
73
|
+
replays the path to the root, and the node above leaf `h` on each level is
|
|
74
|
+
`h >> 3·level`. In a heap's sift-down, each load must wait for the previous
|
|
75
|
+
comparison. Here, the winner among the *other* siblings on each level
|
|
76
|
+
doesn't depend on the value being replayed, so the CPU computes all levels
|
|
77
|
+
in parallel. The serial work per level is a single compare-and-select. On
|
|
78
|
+
aarch64, the 8-way reductions use NEON.
|
|
79
|
+
* **Early exit.** Updates stop once a node's winner is unchanged, so insert,
|
|
80
|
+
decrease-key and changes to non-winning entries usually take one or two
|
|
81
|
+
comparisons.
|
|
82
|
+
* **Order-preserving `u64` keys.** Priorities are stored as integers with the
|
|
83
|
+
same ordering. This gives integer compares and a total order: `-0.0` sorts
|
|
84
|
+
before `+0.0`, and both round-trip exactly. A vacant-leaf sentinel can never
|
|
85
|
+
collide with a real priority, because it corresponds to NaN, which is
|
|
86
|
+
rejected. A max-queue complements the keys.
|
|
87
|
+
* **Growth only appends.** Bulk loads (`push_many`, `PQDict(data)`) rebuild
|
|
88
|
+
the tree in `O(n)`, and building a `PQDict` takes one native call.
|
|
89
|
+
* **Clean layering.** `crates/iheap-core` is pure Rust with no PyO3 dependency
|
|
90
|
+
and has its own tests. `crates/iheap-py` is a thin PyO3 binding.
|
|
91
|
+
`python/iheap/_pqdict.py` implements the mapping API.
|
|
92
|
+
|
|
93
|
+
## API
|
|
94
|
+
|
|
95
|
+
### `PQDict(data=None, *, key=None, reverse=False)`
|
|
96
|
+
|
|
97
|
+
A `MutableMapping` whose items are ordered by priority. The priority is
|
|
98
|
+
`key(value)` if a `key` function is given, otherwise the value itself.
|
|
99
|
+
`reverse=True` makes it a max-queue. Ties are popped in unspecified order.
|
|
100
|
+
|
|
101
|
+
`PQDict` supports **every `dict` operation**: `pq[k]`, `pq[k] = v`,
|
|
102
|
+
`del pq[k]`, `k in pq`, `len`, `iter`, `reversed`, `get`, `setdefault`,
|
|
103
|
+
`pop(k[, default])`, `update`, `|`, `|=`, `keys`/`values`/`items` views,
|
|
104
|
+
`clear`, `copy`, `fromkeys`, equality with any mapping, and pickling. As with
|
|
105
|
+
`dict`, iteration follows **insertion order**. The one intended difference is
|
|
106
|
+
`popitem()`, which removes the **top-priority** item instead of the last one
|
|
107
|
+
inserted.
|
|
108
|
+
|
|
109
|
+
| Method | Description |
|
|
110
|
+
|---|---|
|
|
111
|
+
| `pq[k] = v`, `del pq[k]`, `pq[k]`, `k in pq`, `len(pq)` | mapping protocol; inserts, updates and deletes are `O(log n)` |
|
|
112
|
+
| `top(default)`, `topvalue(default)`, `topitem()` | look at the top item |
|
|
113
|
+
| `pop()`, `popvalue(default)`, `popitem()` | remove the top item (`pop(key[, default])` keeps dict semantics) |
|
|
114
|
+
| `additem(k, v)` / `updateitem(k, v)` | strict insert / strict update |
|
|
115
|
+
| `pushpopitem(k, v)` | push then pop, done as a single tree replay |
|
|
116
|
+
| `replace_key(k, new_k)`, `swap_priority(k1, k2)` | change keys in place (no heap work) |
|
|
117
|
+
| `popkeys()`, `popvalues()`, `popitems()` | destructive iteration in priority order |
|
|
118
|
+
| `heapify(key=None)` | re-prioritize after a value was mutated in place |
|
|
119
|
+
| `update(mapping)` | bulk insert/update with one native call (atomic) |
|
|
120
|
+
| `copy()`, pickling, `fromkeys()` | |
|
|
121
|
+
|
|
122
|
+
Module functions: `minpq(**kw)`, `maxpq(**kw)`, `nsmallest(n, mapping, key=None)`,
|
|
123
|
+
`nlargest(n, mapping, key=None)`.
|
|
124
|
+
|
|
125
|
+
Differences from `pqdict`: priorities must be real numbers, and the custom
|
|
126
|
+
comparator `precedes=` is replaced by `reverse=`.
|
|
127
|
+
|
|
128
|
+
### `IndexedHeap(*, reverse=False, capacity=0)`
|
|
129
|
+
|
|
130
|
+
The raw native heap, for when you want to manage your own key ↔ handle mapping:
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
from iheap import IndexedHeap
|
|
134
|
+
|
|
135
|
+
heap = IndexedHeap()
|
|
136
|
+
h = heap.push(3.5) # -> handle (int)
|
|
137
|
+
heap.update(h, 1.0) # -> old priority
|
|
138
|
+
heap.remove(h) # -> priority
|
|
139
|
+
handles = heap.push_many([5, 2, 8])
|
|
140
|
+
heap.peek() # (handle, priority) or None
|
|
141
|
+
heap.pop() # (handle, priority); IndexError if empty
|
|
142
|
+
heap.pop_many(2)
|
|
143
|
+
heap.push_pop(4.0) # fused push+pop; reuses the popped handle
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
Handles stay valid until their entry is popped or removed, are always below
|
|
147
|
+
`heap.handle_bound`, and are reused afterwards.
|
|
148
|
+
|
|
149
|
+
## Benchmark
|
|
150
|
+
|
|
151
|
+
`benchmarks/bench.py` runs a standard set of indexed-priority-queue
|
|
152
|
+
operations. Each phase starts from fresh state, uses random `float`
|
|
153
|
+
priorities and `str` keys, and reports the best of 5 runs:
|
|
154
|
+
|
|
155
|
+
* **insert**: `n` single inserts into an empty queue
|
|
156
|
+
* **build**: construct from `n` items in one call
|
|
157
|
+
* **update**: `n` priority changes on random existing keys
|
|
158
|
+
* **delete**: `n/2` deletions of random keys
|
|
159
|
+
* **pop**: pop everything in priority order
|
|
160
|
+
|
|
161
|
+
Implementations compared:
|
|
162
|
+
|
|
163
|
+
* `iheap.PQDict`: this library's mapping API
|
|
164
|
+
* `iheap.IndexedHeap`: the raw handle API, with the caller keeping a key → handle `dict`
|
|
165
|
+
* [`pqdict`](https://pypi.org/project/pqdict/) 1.4: the pure-Python reference
|
|
166
|
+
* `heapq (lazy)`: the stdlib "mark removed, skip on pop" recipe from the
|
|
167
|
+
Python docs. Its update and delete are cheap because they defer the work
|
|
168
|
+
to pop.
|
|
169
|
+
|
|
170
|
+
Results on Apple M4 Max, CPython 3.13.7, iheap 0.1.0. Each cell is **ns per
|
|
171
|
+
operation, lower is better**, with the speed-up over `pqdict` in parentheses.
|
|
172
|
+
|
|
173
|
+
**n = 10,000**
|
|
174
|
+
|
|
175
|
+
| implementation | insert | build | update | delete | pop |
|
|
176
|
+
|---|---:|---:|---:|---:|---:|
|
|
177
|
+
| iheap.PQDict | 155 (3.4x) | 48 (11.6x) | 122 (7.1x) | 134 (5.6x) | 139 (16.7x) |
|
|
178
|
+
| iheap.IndexedHeap | 56 (9.4x) | 38 (14.4x) | 55 (15.6x) | 70 (10.8x) | 87 (26.6x) |
|
|
179
|
+
| pqdict | 527 | 552 | 867 | 759 | 2,322 |
|
|
180
|
+
| heapq (lazy) | 185 (2.9x) | 114 (4.9x) | 234 (3.7x) | 76 (9.9x) | 425 (5.5x) |
|
|
181
|
+
|
|
182
|
+
**n = 100,000**
|
|
183
|
+
|
|
184
|
+
| implementation | insert | build | update | delete | pop |
|
|
185
|
+
|---|---:|---:|---:|---:|---:|
|
|
186
|
+
| iheap.PQDict | 178 (3.4x) | 61 (11.1x) | 264 (6.0x) | 317 (4.5x) | 293 (14.9x) |
|
|
187
|
+
| iheap.IndexedHeap | 75 (8.0x) | 53 (12.6x) | 118 (13.5x) | 149 (9.5x) | 184 (23.8x) |
|
|
188
|
+
| pqdict | 603 | 674 | 1,593 | 1,415 | 4,376 |
|
|
189
|
+
| heapq (lazy) | 237 (2.5x) | 147 (4.6x) | 487 (3.3x) | 283 (5.0x) | 970 (4.5x) |
|
|
190
|
+
|
|
191
|
+
**n = 1,000,000**
|
|
192
|
+
|
|
193
|
+
| implementation | insert | build | update | delete | pop |
|
|
194
|
+
|---|---:|---:|---:|---:|---:|
|
|
195
|
+
| iheap.PQDict | 260 (3.0x) | 110 (8.6x) | 660 (4.1x) | 760 (3.3x) | 754 (11.9x) |
|
|
196
|
+
| iheap.IndexedHeap | 151 (5.1x) | 87 (10.9x) | 459 (5.9x) | 491 (5.1x) | 614 (14.6x) |
|
|
197
|
+
| pqdict | 769 | 950 | 2,688 | 2,512 | 8,977 |
|
|
198
|
+
| heapq (lazy) | 360 (2.1x) | 245 (3.9x) | 860 (3.1x) | 552 (4.6x) | 2,473 (3.6x) |
|
|
199
|
+
|
|
200
|
+
### The Rust core alone
|
|
201
|
+
|
|
202
|
+
`cargo run --release --example bench -p iheap-core` measures the core with
|
|
203
|
+
Python out of the picture. The table compares the current tournament tree
|
|
204
|
+
with the 4-ary binary heap used at first, in ns per operation:
|
|
205
|
+
|
|
206
|
+
| n | insert | build | update | delete | pop |
|
|
207
|
+
|---:|---:|---:|---:|---:|---:|
|
|
208
|
+
| 10k | 3.0 → 3.0 (1.0×) | 3.5 → 2.2 (1.6×) | 3.2 → 2.0 (1.6×) | 3.9 → 2.5 (1.6×) | 25.2 → 17.1 (1.5×) |
|
|
209
|
+
| 100k | 5.3 → 4.3 (1.2×) | 4.1 → 2.1 (2.0×) | 10.5 → 3.5 (3.0×) | 5.9 → 3.1 (1.9×) | 37.5 → 20.3 (1.8×) |
|
|
210
|
+
| 1M | 8.0 → 4.9 (1.6×) | 4.7 → 2.6 (1.8×) | 18.2 → 5.6 (3.2×) | 14.1 → 5.4 (2.6×) | 64.0 → 29.6 (2.2×) |
|
|
211
|
+
|
|
212
|
+
From n = 100k up, the core is **2.1× faster** (geometric mean over the five
|
|
213
|
+
operations). In the Python benchmark above, the core accounts for only 5–20%
|
|
214
|
+
of the time; the rest is the interpreter itself (method dispatch, `dict`
|
|
215
|
+
lookups on string keys, object allocation). End to end, the new core makes
|
|
216
|
+
`IndexedHeap` 1.1–1.4× faster and `PQDict` up to 1.3× faster. The
|
|
217
|
+
[benchmark docs](https://nvbkdw.github.io/iheap/benchmark/) include the full
|
|
218
|
+
optimization log, with every experiment that was kept or rejected.
|
|
219
|
+
|
|
220
|
+
Reproduce with:
|
|
221
|
+
|
|
222
|
+
```
|
|
223
|
+
uv venv && uv pip install maturin pqdict
|
|
224
|
+
maturin develop --release
|
|
225
|
+
python benchmarks/bench.py --n 10000 100000 1000000 --markdown
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
## Development
|
|
229
|
+
|
|
230
|
+
```
|
|
231
|
+
cargo test -p iheap-core # Rust core tests (incl. randomized model test)
|
|
232
|
+
maturin develop --release && pytest # Python tests (incl. differential test vs pqdict)
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
Releasing: bump `version` in the root `Cargo.toml`, then push a `vX.Y.Z` tag.
|
|
236
|
+
CI builds the wheels and sdist and publishes them to PyPI through Trusted
|
|
237
|
+
Publishing.
|
|
238
|
+
|
|
239
|
+
## License
|
|
240
|
+
|
|
241
|
+
MIT
|
|
242
|
+
|
iheap-0.1.0/README.md
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
# iheap
|
|
2
|
+
|
|
3
|
+
A fast **indexed priority queue** for Python with a Rust core. It's a drop-in
|
|
4
|
+
style replacement for [`pqdict`](https://github.com/nvictus/pqdict), and is
|
|
5
|
+
**3–20× faster**.
|
|
6
|
+
|
|
7
|
+
```python
|
|
8
|
+
from iheap import PQDict
|
|
9
|
+
|
|
10
|
+
pq = PQDict({"a": 3, "b": 1, "c": 2}) # key -> priority, min on top
|
|
11
|
+
pq["d"] = 0 # insert
|
|
12
|
+
pq["a"] = -1 # update priority in O(log n)
|
|
13
|
+
del pq["c"] # delete any key in O(log n)
|
|
14
|
+
pq.top() # 'a'
|
|
15
|
+
list(pq.popitems()) # [('a', -1), ('d', 0), ('b', 1)]
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
```
|
|
19
|
+
pip install iheap
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
📖 **Documentation:** <https://nvbkdw.github.io/iheap/>. It covers a
|
|
23
|
+
getting-started guide, the API reference, the benchmark and development notes.
|
|
24
|
+
|
|
25
|
+
Prebuilt `abi3` wheels cover CPython ≥ 3.10 on Linux (glibc and musl, x86_64
|
|
26
|
+
and aarch64), macOS (x86_64 and arm64) and Windows (x64).
|
|
27
|
+
|
|
28
|
+
## Design
|
|
29
|
+
|
|
30
|
+
Only priorities (as `f64`) and integer handles cross into Rust. Keys and
|
|
31
|
+
values (the payload) never leave Python.
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
Python (iheap.PQDict) Rust (iheap-core)
|
|
35
|
+
┌───────────────────────────────────┐ ┌─────────────────────────────────────┐
|
|
36
|
+
│ _handles: dict[key -> handle] │ │ 8-ary tournament tree │
|
|
37
|
+
│ _keys: list[handle -> key] │──▶│ leaves[h] = priority of handle h │
|
|
38
|
+
│ _values: list[handle -> value] │ │ node = winner of 8 children │
|
|
39
|
+
└───────────────────────────────────┘ └─────────────────────────────────────┘
|
|
40
|
+
only f64 priorities and u32 handles cross ▲
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
* **Handles, not keys.** The Rust core gives out dense `u32` handles and
|
|
44
|
+
recycles them, so Python keeps its key/value data in plain lists indexed by
|
|
45
|
+
handle. Rust never holds a Python object and never calls back into Python.
|
|
46
|
+
* **An 8-ary tournament (winner) tree, not a binary heap.** Leaf `h` holds the
|
|
47
|
+
priority of handle `h`, so no position map is needed. Each internal node
|
|
48
|
+
stores the winner of 8 children. A sibling group is one 64- or 128-byte
|
|
49
|
+
cache-line block.
|
|
50
|
+
* **The path is known in advance.** Every operation writes one leaf and
|
|
51
|
+
replays the path to the root, and the node above leaf `h` on each level is
|
|
52
|
+
`h >> 3·level`. In a heap's sift-down, each load must wait for the previous
|
|
53
|
+
comparison. Here, the winner among the *other* siblings on each level
|
|
54
|
+
doesn't depend on the value being replayed, so the CPU computes all levels
|
|
55
|
+
in parallel. The serial work per level is a single compare-and-select. On
|
|
56
|
+
aarch64, the 8-way reductions use NEON.
|
|
57
|
+
* **Early exit.** Updates stop once a node's winner is unchanged, so insert,
|
|
58
|
+
decrease-key and changes to non-winning entries usually take one or two
|
|
59
|
+
comparisons.
|
|
60
|
+
* **Order-preserving `u64` keys.** Priorities are stored as integers with the
|
|
61
|
+
same ordering. This gives integer compares and a total order: `-0.0` sorts
|
|
62
|
+
before `+0.0`, and both round-trip exactly. A vacant-leaf sentinel can never
|
|
63
|
+
collide with a real priority, because it corresponds to NaN, which is
|
|
64
|
+
rejected. A max-queue complements the keys.
|
|
65
|
+
* **Growth only appends.** Bulk loads (`push_many`, `PQDict(data)`) rebuild
|
|
66
|
+
the tree in `O(n)`, and building a `PQDict` takes one native call.
|
|
67
|
+
* **Clean layering.** `crates/iheap-core` is pure Rust with no PyO3 dependency
|
|
68
|
+
and has its own tests. `crates/iheap-py` is a thin PyO3 binding.
|
|
69
|
+
`python/iheap/_pqdict.py` implements the mapping API.
|
|
70
|
+
|
|
71
|
+
## API
|
|
72
|
+
|
|
73
|
+
### `PQDict(data=None, *, key=None, reverse=False)`
|
|
74
|
+
|
|
75
|
+
A `MutableMapping` whose items are ordered by priority. The priority is
|
|
76
|
+
`key(value)` if a `key` function is given, otherwise the value itself.
|
|
77
|
+
`reverse=True` makes it a max-queue. Ties are popped in unspecified order.
|
|
78
|
+
|
|
79
|
+
`PQDict` supports **every `dict` operation**: `pq[k]`, `pq[k] = v`,
|
|
80
|
+
`del pq[k]`, `k in pq`, `len`, `iter`, `reversed`, `get`, `setdefault`,
|
|
81
|
+
`pop(k[, default])`, `update`, `|`, `|=`, `keys`/`values`/`items` views,
|
|
82
|
+
`clear`, `copy`, `fromkeys`, equality with any mapping, and pickling. As with
|
|
83
|
+
`dict`, iteration follows **insertion order**. The one intended difference is
|
|
84
|
+
`popitem()`, which removes the **top-priority** item instead of the last one
|
|
85
|
+
inserted.
|
|
86
|
+
|
|
87
|
+
| Method | Description |
|
|
88
|
+
|---|---|
|
|
89
|
+
| `pq[k] = v`, `del pq[k]`, `pq[k]`, `k in pq`, `len(pq)` | mapping protocol; inserts, updates and deletes are `O(log n)` |
|
|
90
|
+
| `top(default)`, `topvalue(default)`, `topitem()` | look at the top item |
|
|
91
|
+
| `pop()`, `popvalue(default)`, `popitem()` | remove the top item (`pop(key[, default])` keeps dict semantics) |
|
|
92
|
+
| `additem(k, v)` / `updateitem(k, v)` | strict insert / strict update |
|
|
93
|
+
| `pushpopitem(k, v)` | push then pop, done as a single tree replay |
|
|
94
|
+
| `replace_key(k, new_k)`, `swap_priority(k1, k2)` | change keys in place (no heap work) |
|
|
95
|
+
| `popkeys()`, `popvalues()`, `popitems()` | destructive iteration in priority order |
|
|
96
|
+
| `heapify(key=None)` | re-prioritize after a value was mutated in place |
|
|
97
|
+
| `update(mapping)` | bulk insert/update with one native call (atomic) |
|
|
98
|
+
| `copy()`, pickling, `fromkeys()` | |
|
|
99
|
+
|
|
100
|
+
Module functions: `minpq(**kw)`, `maxpq(**kw)`, `nsmallest(n, mapping, key=None)`,
|
|
101
|
+
`nlargest(n, mapping, key=None)`.
|
|
102
|
+
|
|
103
|
+
Differences from `pqdict`: priorities must be real numbers, and the custom
|
|
104
|
+
comparator `precedes=` is replaced by `reverse=`.
|
|
105
|
+
|
|
106
|
+
### `IndexedHeap(*, reverse=False, capacity=0)`
|
|
107
|
+
|
|
108
|
+
The raw native heap, for when you want to manage your own key ↔ handle mapping:
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
from iheap import IndexedHeap
|
|
112
|
+
|
|
113
|
+
heap = IndexedHeap()
|
|
114
|
+
h = heap.push(3.5) # -> handle (int)
|
|
115
|
+
heap.update(h, 1.0) # -> old priority
|
|
116
|
+
heap.remove(h) # -> priority
|
|
117
|
+
handles = heap.push_many([5, 2, 8])
|
|
118
|
+
heap.peek() # (handle, priority) or None
|
|
119
|
+
heap.pop() # (handle, priority); IndexError if empty
|
|
120
|
+
heap.pop_many(2)
|
|
121
|
+
heap.push_pop(4.0) # fused push+pop; reuses the popped handle
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Handles stay valid until their entry is popped or removed, are always below
|
|
125
|
+
`heap.handle_bound`, and are reused afterwards.
|
|
126
|
+
|
|
127
|
+
## Benchmark
|
|
128
|
+
|
|
129
|
+
`benchmarks/bench.py` runs a standard set of indexed-priority-queue
|
|
130
|
+
operations. Each phase starts from fresh state, uses random `float`
|
|
131
|
+
priorities and `str` keys, and reports the best of 5 runs:
|
|
132
|
+
|
|
133
|
+
* **insert**: `n` single inserts into an empty queue
|
|
134
|
+
* **build**: construct from `n` items in one call
|
|
135
|
+
* **update**: `n` priority changes on random existing keys
|
|
136
|
+
* **delete**: `n/2` deletions of random keys
|
|
137
|
+
* **pop**: pop everything in priority order
|
|
138
|
+
|
|
139
|
+
Implementations compared:
|
|
140
|
+
|
|
141
|
+
* `iheap.PQDict`: this library's mapping API
|
|
142
|
+
* `iheap.IndexedHeap`: the raw handle API, with the caller keeping a key → handle `dict`
|
|
143
|
+
* [`pqdict`](https://pypi.org/project/pqdict/) 1.4: the pure-Python reference
|
|
144
|
+
* `heapq (lazy)`: the stdlib "mark removed, skip on pop" recipe from the
|
|
145
|
+
Python docs. Its update and delete are cheap because they defer the work
|
|
146
|
+
to pop.
|
|
147
|
+
|
|
148
|
+
Results on Apple M4 Max, CPython 3.13.7, iheap 0.1.0. Each cell is **ns per
|
|
149
|
+
operation, lower is better**, with the speed-up over `pqdict` in parentheses.
|
|
150
|
+
|
|
151
|
+
**n = 10,000**
|
|
152
|
+
|
|
153
|
+
| implementation | insert | build | update | delete | pop |
|
|
154
|
+
|---|---:|---:|---:|---:|---:|
|
|
155
|
+
| iheap.PQDict | 155 (3.4x) | 48 (11.6x) | 122 (7.1x) | 134 (5.6x) | 139 (16.7x) |
|
|
156
|
+
| iheap.IndexedHeap | 56 (9.4x) | 38 (14.4x) | 55 (15.6x) | 70 (10.8x) | 87 (26.6x) |
|
|
157
|
+
| pqdict | 527 | 552 | 867 | 759 | 2,322 |
|
|
158
|
+
| heapq (lazy) | 185 (2.9x) | 114 (4.9x) | 234 (3.7x) | 76 (9.9x) | 425 (5.5x) |
|
|
159
|
+
|
|
160
|
+
**n = 100,000**
|
|
161
|
+
|
|
162
|
+
| implementation | insert | build | update | delete | pop |
|
|
163
|
+
|---|---:|---:|---:|---:|---:|
|
|
164
|
+
| iheap.PQDict | 178 (3.4x) | 61 (11.1x) | 264 (6.0x) | 317 (4.5x) | 293 (14.9x) |
|
|
165
|
+
| iheap.IndexedHeap | 75 (8.0x) | 53 (12.6x) | 118 (13.5x) | 149 (9.5x) | 184 (23.8x) |
|
|
166
|
+
| pqdict | 603 | 674 | 1,593 | 1,415 | 4,376 |
|
|
167
|
+
| heapq (lazy) | 237 (2.5x) | 147 (4.6x) | 487 (3.3x) | 283 (5.0x) | 970 (4.5x) |
|
|
168
|
+
|
|
169
|
+
**n = 1,000,000**
|
|
170
|
+
|
|
171
|
+
| implementation | insert | build | update | delete | pop |
|
|
172
|
+
|---|---:|---:|---:|---:|---:|
|
|
173
|
+
| iheap.PQDict | 260 (3.0x) | 110 (8.6x) | 660 (4.1x) | 760 (3.3x) | 754 (11.9x) |
|
|
174
|
+
| iheap.IndexedHeap | 151 (5.1x) | 87 (10.9x) | 459 (5.9x) | 491 (5.1x) | 614 (14.6x) |
|
|
175
|
+
| pqdict | 769 | 950 | 2,688 | 2,512 | 8,977 |
|
|
176
|
+
| heapq (lazy) | 360 (2.1x) | 245 (3.9x) | 860 (3.1x) | 552 (4.6x) | 2,473 (3.6x) |
|
|
177
|
+
|
|
178
|
+
### The Rust core alone
|
|
179
|
+
|
|
180
|
+
`cargo run --release --example bench -p iheap-core` measures the core with
|
|
181
|
+
Python out of the picture. The table compares the current tournament tree
|
|
182
|
+
with the 4-ary binary heap used at first, in ns per operation:
|
|
183
|
+
|
|
184
|
+
| n | insert | build | update | delete | pop |
|
|
185
|
+
|---:|---:|---:|---:|---:|---:|
|
|
186
|
+
| 10k | 3.0 → 3.0 (1.0×) | 3.5 → 2.2 (1.6×) | 3.2 → 2.0 (1.6×) | 3.9 → 2.5 (1.6×) | 25.2 → 17.1 (1.5×) |
|
|
187
|
+
| 100k | 5.3 → 4.3 (1.2×) | 4.1 → 2.1 (2.0×) | 10.5 → 3.5 (3.0×) | 5.9 → 3.1 (1.9×) | 37.5 → 20.3 (1.8×) |
|
|
188
|
+
| 1M | 8.0 → 4.9 (1.6×) | 4.7 → 2.6 (1.8×) | 18.2 → 5.6 (3.2×) | 14.1 → 5.4 (2.6×) | 64.0 → 29.6 (2.2×) |
|
|
189
|
+
|
|
190
|
+
From n = 100k up, the core is **2.1× faster** (geometric mean over the five
|
|
191
|
+
operations). In the Python benchmark above, the core accounts for only 5–20%
|
|
192
|
+
of the time; the rest is the interpreter itself (method dispatch, `dict`
|
|
193
|
+
lookups on string keys, object allocation). End to end, the new core makes
|
|
194
|
+
`IndexedHeap` 1.1–1.4× faster and `PQDict` up to 1.3× faster. The
|
|
195
|
+
[benchmark docs](https://nvbkdw.github.io/iheap/benchmark/) include the full
|
|
196
|
+
optimization log, with every experiment that was kept or rejected.
|
|
197
|
+
|
|
198
|
+
Reproduce with:
|
|
199
|
+
|
|
200
|
+
```
|
|
201
|
+
uv venv && uv pip install maturin pqdict
|
|
202
|
+
maturin develop --release
|
|
203
|
+
python benchmarks/bench.py --n 10000 100000 1000000 --markdown
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
## Development
|
|
207
|
+
|
|
208
|
+
```
|
|
209
|
+
cargo test -p iheap-core # Rust core tests (incl. randomized model test)
|
|
210
|
+
maturin develop --release && pytest # Python tests (incl. differential test vs pqdict)
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
Releasing: bump `version` in the root `Cargo.toml`, then push a `vX.Y.Z` tag.
|
|
214
|
+
CI builds the wheels and sdist and publishes them to PyPI through Trusted
|
|
215
|
+
Publishing.
|
|
216
|
+
|
|
217
|
+
## License
|
|
218
|
+
|
|
219
|
+
MIT
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "iheap-core"
|
|
3
|
+
description = "Indexed d-ary priority queue over f64 priorities with stable handles"
|
|
4
|
+
version.workspace = true
|
|
5
|
+
edition.workspace = true
|
|
6
|
+
license.workspace = true
|
|
7
|
+
repository.workspace = true
|
|
8
|
+
rust-version.workspace = true
|
|
9
|
+
|
|
10
|
+
[dependencies]
|