fpsketch 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fpsketch/__init__.py +19 -0
- fpsketch/_splitmix.py +28 -0
- fpsketch/mols.py +46 -0
- fpsketch/py.typed +0 -0
- fpsketch/sketching.py +394 -0
- fpsketch-0.1.0.dist-info/METADATA +152 -0
- fpsketch-0.1.0.dist-info/RECORD +9 -0
- fpsketch-0.1.0.dist-info/WHEEL +4 -0
- fpsketch-0.1.0.dist-info/licenses/LICENSE +21 -0
fpsketch/__init__.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""fpsketch: compress molecular count fingerprints to a fixed low dimension
|
|
2
|
+
while preserving Tanimoto (T_MM) similarity as a plain dot product.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
6
|
+
|
|
7
|
+
from .mols import encode_mols
|
|
8
|
+
from .sketching import encode_coo, encode_sparse
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"encode_sparse",
|
|
12
|
+
"encode_coo",
|
|
13
|
+
"encode_mols",
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
try:
|
|
17
|
+
__version__ = version("fpsketch")
|
|
18
|
+
except PackageNotFoundError: # pragma: no cover - not installed, e.g. running from source
|
|
19
|
+
__version__ = "unknown"
|
fpsketch/_splitmix.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""A numpy implementation of splitmix"""
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
U64 = np.uint64
|
|
6
|
+
|
|
7
|
+
# Fixed constants for splitmix
|
|
8
|
+
GAMMA = U64(0x9E3779B97F4A7C15)
|
|
9
|
+
MIX_A = U64(0xBF58476D1CE4E5B9)
|
|
10
|
+
MIX_B = U64(0x94D049BB133111EB)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _splitmix64_hash(x: np.ndarray) -> np.ndarray:
|
|
14
|
+
"""
|
|
15
|
+
Splitmix64 function- an 64 bit rng/hash which is simple to implement,
|
|
16
|
+
invertible, and is considered fairly pseudo-random.
|
|
17
|
+
|
|
18
|
+
It is not strong enough for cryptography but doesn't need to be:
|
|
19
|
+
this is just used to encode fingerprints.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
# Ignore int overflows- this is an intended part of the algorithm
|
|
23
|
+
with np.errstate(over="ignore"):
|
|
24
|
+
z = x + GAMMA
|
|
25
|
+
z = (z ^ (z >> U64(30))) * MIX_A
|
|
26
|
+
z = (z ^ (z >> U64(27))) * MIX_B
|
|
27
|
+
z = z ^ (z >> U64(31))
|
|
28
|
+
return z
|
fpsketch/mols.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""RDKit molecule entry point. Requires the ``chem`` extra (``pip install fpsketch[chem]``)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Sequence
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
|
|
10
|
+
from .sketching import encode_sparse
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def encode_mols(
|
|
14
|
+
mols: Sequence[Any],
|
|
15
|
+
dim: int = 2048,
|
|
16
|
+
generator: Any | None = None,
|
|
17
|
+
num_blocks: int = 4,
|
|
18
|
+
seed: int = 0,
|
|
19
|
+
scale: bool = True,
|
|
20
|
+
) -> np.ndarray:
|
|
21
|
+
"""Sketch RDKit molecules into a dense ``(len(mols), dim)`` array.
|
|
22
|
+
|
|
23
|
+
Args:
|
|
24
|
+
mols: RDKit ``ROMol`` objects.
|
|
25
|
+
dim: output sketch width.
|
|
26
|
+
generator: an ``rdFingerprintGenerator`` generator (Morgan, RDKitFP,
|
|
27
|
+
AtomPair, TopologicalTorsion, ...). Defaults to
|
|
28
|
+
``GetMorganGenerator(radius=2)`` if not given. Any generator
|
|
29
|
+
exposing ``GetSparseCountFingerprint`` works.
|
|
30
|
+
num_blocks: see ``encode_sparse``.
|
|
31
|
+
seed: hash seed; two sketches are only dot-product-comparable if built
|
|
32
|
+
with the same seed.
|
|
33
|
+
scale: see ``encode_sparse``.
|
|
34
|
+
"""
|
|
35
|
+
try:
|
|
36
|
+
from rdkit.Chem import rdFingerprintGenerator
|
|
37
|
+
except ImportError as e:
|
|
38
|
+
raise ImportError(
|
|
39
|
+
"encode_mols requires rdkit. Install it with `pip install fpsketch[chem]`."
|
|
40
|
+
) from e
|
|
41
|
+
|
|
42
|
+
if generator is None:
|
|
43
|
+
generator = rdFingerprintGenerator.GetMorganGenerator(radius=2)
|
|
44
|
+
|
|
45
|
+
fps = [generator.GetSparseCountFingerprint(mol).GetNonzeroElements() for mol in mols]
|
|
46
|
+
return encode_sparse(fps, dim=dim, num_blocks=num_blocks, seed=seed, scale=scale)
|
fpsketch/py.typed
ADDED
|
File without changes
|
fpsketch/sketching.py
ADDED
|
@@ -0,0 +1,394 @@
|
|
|
1
|
+
"""Sparse count-fingerprint sketching.
|
|
2
|
+
|
|
3
|
+
Hashes each molecule's unary-encoded count fingerprint into a fixed-width
|
|
4
|
+
dense vector such that ``encode_sparse(...) @ encode_sparse(...).T`` approximates
|
|
5
|
+
``T_MM`` (min-max Tanimoto similarity) as a plain dot product.
|
|
6
|
+
|
|
7
|
+
This is a block-disjoint sparse Johnson-Lindenstrauss sketch (`num_blocks`
|
|
8
|
+
independent CountSketches, each writing into its own disjoint slice of the output,
|
|
9
|
+
averaged for lower-variance dot products): each unary level of a feature's count
|
|
10
|
+
gets its own hashed column and sign, so the unary decomposition happens
|
|
11
|
+
implicitly inside the hash key rather than as a separate materialized step.
|
|
12
|
+
|
|
13
|
+
Two entry points share this pipeline: ``encode_sparse`` takes one
|
|
14
|
+
``{feature_id: count}`` dict per molecule, while ``encode_coo`` takes an
|
|
15
|
+
already-vectorized COO-format sparse array (e.g. ``scipy.sparse.coo_array``,
|
|
16
|
+
or anything exposing ``.row``/``.col``/``.data``/``.shape``) directly, skipping
|
|
17
|
+
the per-molecule dict traversal for callers who already have their counts in
|
|
18
|
+
that form.
|
|
19
|
+
|
|
20
|
+
Hashing is vectorized: `hashlib` is used only once per call, in `_subkeys`, to
|
|
21
|
+
turn the user's seed into a handful of subkeys. Per-element hashing is done with
|
|
22
|
+
a pure-numpy splitmix64 mixer instead of `hashlib` per element.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import hashlib
|
|
28
|
+
import struct
|
|
29
|
+
from collections.abc import Mapping, Sequence
|
|
30
|
+
from typing import Any
|
|
31
|
+
|
|
32
|
+
import numpy as np
|
|
33
|
+
|
|
34
|
+
from ._splitmix import GAMMA as SPLITMIX64_GAMMA
|
|
35
|
+
from ._splitmix import _splitmix64_hash
|
|
36
|
+
|
|
37
|
+
U64 = np.uint64
|
|
38
|
+
|
|
39
|
+
_EMPTY_COO = (
|
|
40
|
+
np.empty(0, dtype=np.int64),
|
|
41
|
+
np.empty(0, dtype=U64),
|
|
42
|
+
np.empty(0, dtype=np.int64),
|
|
43
|
+
)
|
|
44
|
+
_EMPTY_UNARY = (
|
|
45
|
+
np.empty(0, dtype=np.int64),
|
|
46
|
+
np.empty(0, dtype=U64),
|
|
47
|
+
np.empty(0, dtype=U64),
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _coo_from_fps(
|
|
52
|
+
fps: Sequence[Mapping[int, int]],
|
|
53
|
+
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
|
54
|
+
"""Flatten sparse count fingerprints into parallel COO ``(row, id, count)`` arrays.
|
|
55
|
+
|
|
56
|
+
This is the same ``(row, col, data)`` shape as a COO sparse matrix of
|
|
57
|
+
molecules x features, just not wrapped in a sparse-array class. Callers who
|
|
58
|
+
already have that shape (e.g. a ``scipy.sparse.coo_array``) can skip this
|
|
59
|
+
function entirely and go straight to ``encode_coo``.
|
|
60
|
+
|
|
61
|
+
Args:
|
|
62
|
+
fps: one ``{feature_id: count}`` mapping per molecule. Feature ids must be
|
|
63
|
+
non-negative (raises ``ValueError`` otherwise); entries with a count
|
|
64
|
+
of zero or less are dropped.
|
|
65
|
+
|
|
66
|
+
Returns:
|
|
67
|
+
Three parallel 1-D arrays, each of length ``nnz`` (the number of
|
|
68
|
+
surviving, positive-count entries):
|
|
69
|
+
|
|
70
|
+
- ``rows`` (int64): which molecule the entry came from, an index into ``fps``.
|
|
71
|
+
- ``ids`` (uint64): which feature the entry came from.
|
|
72
|
+
- ``counts`` (int64): the (positive) count for that ``(row, id)`` pair.
|
|
73
|
+
|
|
74
|
+
Example:
|
|
75
|
+
>>> rows, ids, counts = _coo_from_fps([{7: 2, 9: 1}, {}, {4: 3}])
|
|
76
|
+
>>> rows
|
|
77
|
+
array([0, 0, 2])
|
|
78
|
+
>>> ids
|
|
79
|
+
array([7, 9, 4], dtype=uint64)
|
|
80
|
+
>>> counts
|
|
81
|
+
array([2, 1, 3])
|
|
82
|
+
"""
|
|
83
|
+
row_chunks, feat_chunks, count_chunks = [], [], []
|
|
84
|
+
for row, fp in enumerate(fps):
|
|
85
|
+
if not fp:
|
|
86
|
+
continue
|
|
87
|
+
min_id = min(fp)
|
|
88
|
+
if min_id < 0:
|
|
89
|
+
raise ValueError(
|
|
90
|
+
"encode_sparse: feature ids must be non-negative; got "
|
|
91
|
+
f"{min_id!r} in fingerprint at index {row}."
|
|
92
|
+
)
|
|
93
|
+
feat_chunks.append(np.fromiter(fp.keys(), dtype=U64, count=len(fp)))
|
|
94
|
+
count_chunks.append(np.fromiter(fp.values(), dtype=np.int64, count=len(fp)))
|
|
95
|
+
row_chunks.append(np.full(len(fp), row, dtype=np.int64))
|
|
96
|
+
|
|
97
|
+
if not feat_chunks:
|
|
98
|
+
return _EMPTY_COO
|
|
99
|
+
|
|
100
|
+
ids = np.concatenate(feat_chunks)
|
|
101
|
+
counts = np.concatenate(count_chunks)
|
|
102
|
+
rows = np.concatenate(row_chunks)
|
|
103
|
+
|
|
104
|
+
keep = counts > 0
|
|
105
|
+
return rows[keep], ids[keep], counts[keep]
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _unary_expand(
|
|
109
|
+
rows: np.ndarray, ids: np.ndarray, counts: np.ndarray
|
|
110
|
+
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
|
111
|
+
"""Expand COO ``(row, id, count)`` triples into flat unary ``(row, id, level)`` triples.
|
|
112
|
+
|
|
113
|
+
The unary encoding represents a count ``c`` by the indicators
|
|
114
|
+
``1[c > 0], 1[c > 1], ..., 1[c > c - 1]`` -- i.e. ``c`` levels, numbered
|
|
115
|
+
``0`` to ``c - 1``. This function materializes one entry per level, but
|
|
116
|
+
only as three flat index arrays: the unary expansion's own (enormous)
|
|
117
|
+
dimension is never allocated, and the arrays are exactly the input a
|
|
118
|
+
vectorized CountSketch needs.
|
|
119
|
+
|
|
120
|
+
Args:
|
|
121
|
+
rows, ids, counts: parallel COO arrays as returned by ``_coo_from_fps``
|
|
122
|
+
(or built directly from a caller's sparse array in ``encode_coo``).
|
|
123
|
+
``counts`` must already be strictly positive -- callers are
|
|
124
|
+
responsible for dropping zero/negative entries before this point.
|
|
125
|
+
|
|
126
|
+
Returns:
|
|
127
|
+
Three parallel 1-D arrays, each of length ``nnz`` (the total count mass,
|
|
128
|
+
``counts.sum()``):
|
|
129
|
+
|
|
130
|
+
- ``out_rows`` (int64): which molecule the level came from.
|
|
131
|
+
- ``out_ids`` (uint64): which feature the level came from.
|
|
132
|
+
- ``levels`` (uint64): which level this is within its feature, ``0 .. count - 1``.
|
|
133
|
+
|
|
134
|
+
Example:
|
|
135
|
+
>>> rows, ids, levels = _unary_expand(
|
|
136
|
+
... np.array([0, 0, 2]), np.array([7, 9, 4], dtype=np.uint64), np.array([2, 1, 3])
|
|
137
|
+
... )
|
|
138
|
+
>>> rows
|
|
139
|
+
array([0, 0, 0, 2, 2, 2])
|
|
140
|
+
>>> ids
|
|
141
|
+
array([7, 7, 9, 4, 4, 4], dtype=uint64)
|
|
142
|
+
>>> levels
|
|
143
|
+
array([0, 1, 0, 0, 1, 2], dtype=uint64)
|
|
144
|
+
"""
|
|
145
|
+
if not counts.size:
|
|
146
|
+
return _EMPTY_UNARY
|
|
147
|
+
|
|
148
|
+
# One entry per level: feature f with count c is repeated c times.
|
|
149
|
+
out_rows = np.repeat(rows, counts)
|
|
150
|
+
out_ids = np.repeat(ids, counts)
|
|
151
|
+
|
|
152
|
+
# Levels are a "ragged arange": 0..c-1 restarting at every feature. Globally
|
|
153
|
+
# that is arange(nnz) minus the position at which each entry's run began.
|
|
154
|
+
run_starts = np.repeat(np.cumsum(counts) - counts, counts)
|
|
155
|
+
levels = np.arange(len(out_rows), dtype=np.int64) - run_starts
|
|
156
|
+
|
|
157
|
+
return out_rows, out_ids, levels.astype(U64)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _subkeys(seed: int, n: int) -> np.ndarray:
|
|
161
|
+
"""Derive ``n`` independent 64 bit ints from a single seed.
|
|
162
|
+
|
|
163
|
+
Done by hashing the seed interpreting the hash digest as a stream of ints.
|
|
164
|
+
The shake_256 hash is used because it can produce an arbitrarily long digest.
|
|
165
|
+
|
|
166
|
+
Args:
|
|
167
|
+
seed: the user's seed (int)
|
|
168
|
+
n: number of uint64 subkeys to derive.
|
|
169
|
+
|
|
170
|
+
Returns:
|
|
171
|
+
A ``(n,)`` uint64 array.
|
|
172
|
+
|
|
173
|
+
Example:
|
|
174
|
+
>>> keys = _subkeys(0, 4)
|
|
175
|
+
>>> keys.dtype
|
|
176
|
+
dtype('uint64')
|
|
177
|
+
>>> len(keys) == len(set(keys.tolist()))
|
|
178
|
+
True
|
|
179
|
+
"""
|
|
180
|
+
seed_as_bytes = struct.pack("<q", seed)
|
|
181
|
+
digest = hashlib.shake_256(seed_as_bytes).digest(8 * n)
|
|
182
|
+
return np.frombuffer(digest, dtype="<u8").copy()
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def encode_sparse(
|
|
186
|
+
fps: Sequence[Mapping[int, int]],
|
|
187
|
+
dim: int,
|
|
188
|
+
num_blocks: int = 4,
|
|
189
|
+
seed: int = 0,
|
|
190
|
+
scale: bool = True,
|
|
191
|
+
) -> np.ndarray:
|
|
192
|
+
"""Sketch sparse count fingerprints into a dense ``(len(fps), dim)`` array.
|
|
193
|
+
|
|
194
|
+
Args:
|
|
195
|
+
fps: one ``{feature_id: count}`` mapping per molecule (e.g. RDKit's
|
|
196
|
+
``GetSparseCountFingerprint(mol).GetNonzeroElements()``). Feature
|
|
197
|
+
ids must be non-negative; entries with a count of zero or less
|
|
198
|
+
are dropped.
|
|
199
|
+
dim: output sketch width.
|
|
200
|
+
num_blocks: number of disjoint sparse-JL blocks to split ``dim`` into.
|
|
201
|
+
More blocks trade a small amount of dot-product accuracy for
|
|
202
|
+
better tail concentration; the default of 4 is a reasonable
|
|
203
|
+
general-purpose choice. Must be at least 1 and at most ``dim``.
|
|
204
|
+
seed: hash seed; two sketches are only dot-product-comparable if built
|
|
205
|
+
with the same seed.
|
|
206
|
+
scale: if True (default), divide the output by ``sqrt(num_blocks)`` so
|
|
207
|
+
that ``sketch @ sketch.T`` and ``(sketch**2).sum(axis=1)`` directly
|
|
208
|
+
approximate the true (unnormalized) dot product / count mass of the
|
|
209
|
+
original unary encoding. This factor cancels out of the ``T_DP``
|
|
210
|
+
ratio (``s.s' / (||s||^2 + ||s'||^2 - s.s')``), so callers who only
|
|
211
|
+
ever compute Tanimoto similarity through that ratio can safely set
|
|
212
|
+
this to False and skip the extra pass over the output array; anyone
|
|
213
|
+
comparing raw dot products/norms, including across different
|
|
214
|
+
``num_blocks`` settings, should leave it True.
|
|
215
|
+
|
|
216
|
+
Returns:
|
|
217
|
+
A ``(len(fps), dim)`` float64 array. Its rows' pairwise dot products
|
|
218
|
+
approximate ``T_DP``, which equals ``T_MM`` on the original counts.
|
|
219
|
+
"""
|
|
220
|
+
rows, ids, counts = _coo_from_fps(fps)
|
|
221
|
+
return _encode_core(rows, ids, counts, len(fps), dim, num_blocks, seed, scale)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def encode_coo(
|
|
225
|
+
coo: Any,
|
|
226
|
+
dim: int,
|
|
227
|
+
num_blocks: int = 4,
|
|
228
|
+
seed: int = 0,
|
|
229
|
+
scale: bool = True,
|
|
230
|
+
) -> np.ndarray:
|
|
231
|
+
"""Sketch an already-vectorized COO sparse count array into ``(n_rows, dim)``.
|
|
232
|
+
|
|
233
|
+
This is the same computation as ``encode_sparse``, but for callers who
|
|
234
|
+
already have their molecules x features counts in COO form (e.g. a
|
|
235
|
+
``scipy.sparse.coo_array``) and want to skip the per-molecule dict
|
|
236
|
+
traversal ``encode_sparse`` does internally to get there.
|
|
237
|
+
|
|
238
|
+
Args:
|
|
239
|
+
coo: a COO-format sparse array-like object exposing ``.row`` (molecule
|
|
240
|
+
index per entry), ``.col`` (feature id per entry), ``.data`` (count
|
|
241
|
+
per entry), and ``.shape`` (``(n_rows, n_features)``) -- e.g. a
|
|
242
|
+
``scipy.sparse.coo_array``. Feature ids (``.col``) must be
|
|
243
|
+
non-negative; entries with a count of zero or less are dropped.
|
|
244
|
+
dim: output sketch width.
|
|
245
|
+
num_blocks: see ``encode_sparse``.
|
|
246
|
+
seed: see ``encode_sparse``.
|
|
247
|
+
scale: see ``encode_sparse``.
|
|
248
|
+
|
|
249
|
+
Returns:
|
|
250
|
+
A ``(coo.shape[0], dim)`` float64 array, matching ``encode_sparse``'s
|
|
251
|
+
output for the equivalent dict-of-counts input.
|
|
252
|
+
"""
|
|
253
|
+
for attr in ("row", "col", "data", "shape"):
|
|
254
|
+
if not hasattr(coo, attr):
|
|
255
|
+
raise TypeError(
|
|
256
|
+
"encode_coo expects a COO-format sparse array-like object "
|
|
257
|
+
"(e.g. scipy.sparse.coo_array) exposing .row, .col, .data, and "
|
|
258
|
+
f".shape; got {type(coo).__name__!r}, missing '{attr}'."
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
n_rows = coo.shape[0]
|
|
262
|
+
rows = np.asarray(coo.row, dtype=np.int64)
|
|
263
|
+
ids = np.asarray(coo.col)
|
|
264
|
+
counts = np.asarray(coo.data, dtype=np.int64)
|
|
265
|
+
|
|
266
|
+
if ids.size and ids.min() < 0:
|
|
267
|
+
raise ValueError("encode_coo: column indices (feature ids) must be non-negative.")
|
|
268
|
+
ids = ids.astype(U64)
|
|
269
|
+
|
|
270
|
+
keep = counts > 0
|
|
271
|
+
rows, ids, counts = rows[keep], ids[keep], counts[keep]
|
|
272
|
+
|
|
273
|
+
return _encode_core(rows, ids, counts, n_rows, dim, num_blocks, seed, scale)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _encode_core(
|
|
277
|
+
rows: np.ndarray,
|
|
278
|
+
ids: np.ndarray,
|
|
279
|
+
counts: np.ndarray,
|
|
280
|
+
n_rows: int,
|
|
281
|
+
dim: int,
|
|
282
|
+
num_blocks: int,
|
|
283
|
+
seed: int,
|
|
284
|
+
scale: bool,
|
|
285
|
+
) -> np.ndarray:
|
|
286
|
+
"""Shared sketch pipeline for ``encode_sparse`` and ``encode_coo``.
|
|
287
|
+
|
|
288
|
+
NOTE: for speed this algorithm implements custom hashing in numpy instead
|
|
289
|
+
of using python's standard library (e.g. hashlib). The motivation for each
|
|
290
|
+
step is explained inline in the comments.
|
|
291
|
+
|
|
292
|
+
Args:
|
|
293
|
+
rows, ids, counts: parallel COO arrays (molecule index, feature id,
|
|
294
|
+
positive count), as returned by ``_coo_from_fps`` or built directly
|
|
295
|
+
in ``encode_coo``.
|
|
296
|
+
n_rows: number of molecules (the output's first dimension).
|
|
297
|
+
dim, num_blocks, seed, scale: see ``encode_sparse``.
|
|
298
|
+
"""
|
|
299
|
+
if num_blocks < 1 or num_blocks > dim:
|
|
300
|
+
raise ValueError(f"num_blocks must be between 1 and dim (dim={dim}); got {num_blocks}")
|
|
301
|
+
|
|
302
|
+
# Step 0: Transform data to vectorized format.
|
|
303
|
+
# These are 1D arrays; shape is nnz in the dataset.
|
|
304
|
+
# Use to initialize the output array
|
|
305
|
+
rows, feature_ids, count_levels = _unary_expand(rows, ids, counts)
|
|
306
|
+
out = np.zeros((n_rows, dim))
|
|
307
|
+
|
|
308
|
+
# Hash step 1: create a single 64 bit hash of the
|
|
309
|
+
# (feature_ids, count_levels) tuples. We call it "fp_keys".
|
|
310
|
+
# It gets combined with another hash of the (seed, function)
|
|
311
|
+
# tuple later.
|
|
312
|
+
#
|
|
313
|
+
# Step 1a: hash feature_ids only with splitmix. There is no
|
|
314
|
+
# guarantee that feature_ids are uniformly distributed across
|
|
315
|
+
# 64 bits (they could be a hand constructed fingerprint with ids 0, 1, 2, etc).
|
|
316
|
+
# We apply splitmix once to hopefully disperse them uniformly
|
|
317
|
+
fp_keys = _splitmix64_hash(feature_ids) # maps feature_ids
|
|
318
|
+
|
|
319
|
+
# Step 1b: add an offset from count_levels. The scaling constant
|
|
320
|
+
# SPLITMIX64_GAMMA is a canonical shift constant from the splitmix
|
|
321
|
+
# algorithm, designed to cycle through all 2^64 ints before repeating,
|
|
322
|
+
# so identical fp_key values with different levels will get mapped
|
|
323
|
+
# to different bits (unless counts are > 2^64, which is not actually supported).
|
|
324
|
+
# We ignore overflow errors because overflow is intended (technically it should
|
|
325
|
+
# be addition mod 2^64, but overflow handles this for us).
|
|
326
|
+
#
|
|
327
|
+
# After this step, we consider the (feature_id, count_level) pairs to
|
|
328
|
+
# be jointly hashed and no longer refer to these features individually.
|
|
329
|
+
with np.errstate(over="ignore"):
|
|
330
|
+
fp_keys = fp_keys + count_levels * SPLITMIX64_GAMMA
|
|
331
|
+
|
|
332
|
+
# Hash step 2: get two 64 bit ints for each block: one for the column hash
|
|
333
|
+
# and one for the sign hash. We call the _subkeys method for this, which
|
|
334
|
+
# generates these pseudo-randomly from the input seed. Call it "seed_block_keys"
|
|
335
|
+
# for maximum clarity.
|
|
336
|
+
seed_block_keys = _subkeys(seed=seed, n=2 * num_blocks)
|
|
337
|
+
|
|
338
|
+
# Main loop over blocks: we will now separately fill in each block
|
|
339
|
+
standard_block_length = dim // num_blocks # round down, last block will get remainder
|
|
340
|
+
for block_idx in range(num_blocks):
|
|
341
|
+
# Get block start and end indices
|
|
342
|
+
block_start = block_idx * standard_block_length
|
|
343
|
+
this_block_length = standard_block_length
|
|
344
|
+
if block_idx == num_blocks - 1:
|
|
345
|
+
this_block_length += dim % num_blocks # add remainder to last block
|
|
346
|
+
|
|
347
|
+
# Hash step 3: XOR fp_keys with the appropriate seed_block_keys.
|
|
348
|
+
# The function f(a) = a XOR b (for a constant b) is bijective,
|
|
349
|
+
# so distinct fp_keys are guaranteed to map to distinct outputs.
|
|
350
|
+
# We call these "pre*" because we will hash again in step 4
|
|
351
|
+
this_block_col_key = seed_block_keys[2 * block_idx]
|
|
352
|
+
this_block_sign_key = seed_block_keys[2 * block_idx + 1]
|
|
353
|
+
pre_col_fp_hash = fp_keys ^ this_block_col_key
|
|
354
|
+
pre_sign_fp_hash = fp_keys ^ this_block_sign_key
|
|
355
|
+
|
|
356
|
+
# Hash step 4: although we could use the pre-hashes directly
|
|
357
|
+
# as the hash, they are arguably not independent enough between
|
|
358
|
+
# blocks. For example, we will only use one bit of the sign hash,
|
|
359
|
+
# and if that bit happens to match between 2 block keys then *all*
|
|
360
|
+
# the signs will match between those two blocks. To avoid this,
|
|
361
|
+
# we re-hash the pre-hashes with splitmix, so differences cascade
|
|
362
|
+
# more uniformly to all the bits.
|
|
363
|
+
col_hash = _splitmix64_hash(pre_col_fp_hash)
|
|
364
|
+
sign_hash = _splitmix64_hash(pre_sign_fp_hash)
|
|
365
|
+
|
|
366
|
+
# Pick a col: col_hash mod block length.
|
|
367
|
+
# intp dtype is meant for array indexing.
|
|
368
|
+
# NOTE: col is relative to the block start- we apply this offset later
|
|
369
|
+
col = (col_hash % U64(this_block_length)).astype(np.intp)
|
|
370
|
+
|
|
371
|
+
# Pick a sign: take the first bit. We do this by bit-shifting by 63
|
|
372
|
+
# and AND-ing with 1 (just to be safe).
|
|
373
|
+
# 0 means "-1" and 1 means 1- we shift this in the next step
|
|
374
|
+
sign01 = (sign_hash >> U64(63)) & U64(1)
|
|
375
|
+
sign = 2 * sign01.astype(out.dtype) - 1
|
|
376
|
+
|
|
377
|
+
# We will now add entries to out using np.bincount. However, this
|
|
378
|
+
# accepts a 1D output, so we need to flatten and unflatten
|
|
379
|
+
fill_indices = rows * this_block_length + col # indices to fill in flattened array
|
|
380
|
+
fill_values = np.bincount(
|
|
381
|
+
fill_indices, # we are counting every time each index is hit
|
|
382
|
+
weights=sign, # by default it adds 1, we override to add either +1 or -1
|
|
383
|
+
minlength=n_rows * this_block_length, # ensure full output dimension
|
|
384
|
+
)
|
|
385
|
+
out[:, block_start : block_start + this_block_length] = fill_values.reshape(
|
|
386
|
+
n_rows, this_block_length
|
|
387
|
+
)
|
|
388
|
+
|
|
389
|
+
# Before returning, scale out by the number of blocks (see the `scale` docstring
|
|
390
|
+
# on encode_sparse: this cancels out of the T_DP ratio, but keeps raw dot
|
|
391
|
+
# products/norms comparable across different num_blocks settings).
|
|
392
|
+
if scale:
|
|
393
|
+
out = out / np.sqrt(num_blocks)
|
|
394
|
+
return out
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: fpsketch
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Compress molecular count fingerprints to a fixed low dimension while preserving Tanimoto similarity as a plain dot product.
|
|
5
|
+
Project-URL: Homepage, https://github.com/AustinT/fpsketch
|
|
6
|
+
Project-URL: Repository, https://github.com/AustinT/fpsketch
|
|
7
|
+
Author: Austin Tripp
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: cheminformatics,fingerprint,johnson-lindenstrauss,sketching,tanimoto
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering :: Chemistry
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Requires-Dist: numpy>=1.24
|
|
18
|
+
Provides-Extra: chem
|
|
19
|
+
Requires-Dist: rdkit>=2023.9; extra == 'chem'
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
|
|
22
|
+
# fpsketch
|
|
23
|
+
|
|
24
|
+
Compress molecular count fingerprints to a fixed low dimension while
|
|
25
|
+
(approximately) preserving Tanimoto similarity as a plain dot product.
|
|
26
|
+
|
|
27
|
+
Given two count fingerprints `x`, `x'` (e.g. Morgan fingerprints with counts,
|
|
28
|
+
`{feature_id: count}`), the standard chemistry similarity metric is the
|
|
29
|
+
min-max Tanimoto:
|
|
30
|
+
|
|
31
|
+
```
|
|
32
|
+
T_MM(x, x') = sum_i min(x_i, x'_i) / sum_i max(x_i, x'_i)
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
`T_MM` isn't a dot product, so you can't drop count fingerprints straight into
|
|
36
|
+
models (nearest-neighbor search, GPs, kernel methods, ...) that expect a plain
|
|
37
|
+
inner product. fpsketch sketches count fingerprints into a fixed-width dense
|
|
38
|
+
vector `s = encode(x)` such that
|
|
39
|
+
|
|
40
|
+
```
|
|
41
|
+
T_DP(s, s') = s.s' / (||s||^2 + ||s'||^2 - s.s') ≈ T_MM(x, x')
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
i.e. an ordinary dot product on the sketch approximates `T_MM` on the
|
|
45
|
+
original fingerprints.
|
|
46
|
+
|
|
47
|
+
## Why this works
|
|
48
|
+
|
|
49
|
+
A unary encoding turns each count into a set of indicators,
|
|
50
|
+
`psi(x)_{i,k} = 1[x_i > k]` for `k = 0 .. x_i - 1`. Since
|
|
51
|
+
`min(u, v) = sum_k 1[u > k] * 1[v > k]`, this makes `T_DP(psi(x), psi(x'))`
|
|
52
|
+
exactly equal to `T_MM(x, x')` -- no approximation yet, just a reformulation.
|
|
53
|
+
fpsketch then applies a CountSketch (hashing each `(feature_id, level)` pair
|
|
54
|
+
into one of `m` signed buckets) to that unary expansion, which keeps the
|
|
55
|
+
dot product unbiased while collapsing it to a fixed, low dimension.
|
|
56
|
+
|
|
57
|
+
## Install
|
|
58
|
+
|
|
59
|
+
```
|
|
60
|
+
pip install fpsketch # encode_sparse only, numpy-only
|
|
61
|
+
pip install fpsketch[chem] # + encode_mols, pulls in rdkit
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## Quickstart
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
from fpsketch import encode_sparse
|
|
68
|
+
|
|
69
|
+
# Sparse count fingerprints you already have.
|
|
70
|
+
fps = [{1: 1, 2: 2, 3: 3}, {4: 4, 5: 5, 6: 6}]
|
|
71
|
+
sketch = encode_sparse(fps, dim=2048, seed=0)
|
|
72
|
+
|
|
73
|
+
# T_DP as a plain dot product / normalized similarity.
|
|
74
|
+
G = sketch @ sketch.T
|
|
75
|
+
sq = (sketch**2).sum(axis=1)
|
|
76
|
+
similarity = G / (sq[:, None] + sq[None, :] - G)
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
from rdkit import Chem
|
|
81
|
+
from fpsketch import encode_mols
|
|
82
|
+
|
|
83
|
+
mols = [Chem.MolFromSmiles(s) for s in ["CCO"]] # your list of SMILES strings
|
|
84
|
+
sketch = encode_mols(mols, dim=2048, seed=0) # defaults to a Morgan(radius=2) generator
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
If you already have your counts vectorized as a COO sparse array (molecules x
|
|
88
|
+
features), `encode_coo` skips the per-molecule dict traversal `encode_sparse`
|
|
89
|
+
does internally:
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
from scipy.sparse import coo_array
|
|
93
|
+
from fpsketch import encode_coo
|
|
94
|
+
|
|
95
|
+
counts = coo_array(...) # shape (n_molecules, n_features)
|
|
96
|
+
sketch = encode_coo(counts, dim=2048, seed=0)
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Two sketches are only comparable if built with the same `seed`.
|
|
100
|
+
|
|
101
|
+
## Choosing `dim` and `num_blocks`
|
|
102
|
+
|
|
103
|
+
`dim=2048` is a strong default for typical fingerprint settings. For extra
|
|
104
|
+
safety margin, `dim` at 2-4x the fingerprint's effective (unfolded) dimension
|
|
105
|
+
is a reasonable range to sweep. `num_blocks` (default 4) splits `dim` into
|
|
106
|
+
that many disjoint sub-sketches, each an independent CountSketch; a dot
|
|
107
|
+
product on the concatenated output is equivalent to averaging the
|
|
108
|
+
`num_blocks` per-block dot-product estimates. This trades a small amount of
|
|
109
|
+
raw accuracy for better tail concentration across single-draw sketches, which
|
|
110
|
+
matters when a sketch is computed once and fed straight into a downstream
|
|
111
|
+
model (e.g. a GP) rather than averaged over many random seeds.
|
|
112
|
+
|
|
113
|
+
## The `scale` parameter
|
|
114
|
+
|
|
115
|
+
By default (`scale=True`), the output is divided by `sqrt(num_blocks)` so
|
|
116
|
+
that raw dot products and squared norms directly approximate the true,
|
|
117
|
+
unnormalized dot product / count mass of the original fingerprints -- useful
|
|
118
|
+
if you compare sketches built with different `num_blocks`, or use the sketch
|
|
119
|
+
for anything beyond the `T_DP` ratio above (cosine similarity, nearest
|
|
120
|
+
neighbors on raw dot product, etc). That factor cancels out of the `T_DP`
|
|
121
|
+
ratio itself, so if you only ever compute Tanimoto similarity through that
|
|
122
|
+
ratio, `scale=False` is equivalent and skips one pass over the output array.
|
|
123
|
+
|
|
124
|
+
## Performance note
|
|
125
|
+
|
|
126
|
+
Hashing is vectorized with numpy (a pure-numpy splitmix64 mixer, not
|
|
127
|
+
`hashlib` per element) rather than hashing one `(feature, count-level)` pair
|
|
128
|
+
at a time -- see `src/fpsketch/sketching.py` for details.
|
|
129
|
+
|
|
130
|
+
## Development
|
|
131
|
+
|
|
132
|
+
```
|
|
133
|
+
uv sync --extra chem
|
|
134
|
+
uv run pytest
|
|
135
|
+
uv run ruff check .
|
|
136
|
+
uv run ruff format .
|
|
137
|
+
uv run mypy src
|
|
138
|
+
uv run pre-commit install # run the above automatically on each commit
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
## Releasing
|
|
142
|
+
|
|
143
|
+
Versions are derived from git tags (`hatch-vcs`); there is no version to bump
|
|
144
|
+
by hand. Pushing a tag matching `v*` (e.g. `v0.2.0`) triggers
|
|
145
|
+
`.github/workflows/publish.yml`, which builds and publishes to PyPI via
|
|
146
|
+
[Trusted Publishing](https://docs.pypi.org/trusted-publishers/) -- no API
|
|
147
|
+
token needed, but the `pypi` environment must be configured as a trusted
|
|
148
|
+
publisher for this repo in the PyPI project settings first.
|
|
149
|
+
|
|
150
|
+
## License
|
|
151
|
+
|
|
152
|
+
MIT, see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
fpsketch/__init__.py,sha256=RH7er_mlowsnx43DRXJ5xg3myDPDm5w0wpNoLzOeDd8,526
|
|
2
|
+
fpsketch/_splitmix.py,sha256=pdidDzt4Q0XpOsUqkX27RI5w1wUEhtjOcTTsJ4ajc5g,777
|
|
3
|
+
fpsketch/mols.py,sha256=RFo0hWigMGM61pcwsitQR32PUyRIBbV3riu3tCAOdNo,1558
|
|
4
|
+
fpsketch/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
fpsketch/sketching.py,sha256=Y8EETmg94p2b7APKkaQm_ChL3fErgOoRGndjHfgOSHI,16676
|
|
6
|
+
fpsketch-0.1.0.dist-info/METADATA,sha256=KaJxISNWj4cHTz3ipMbzupgp_9lKZrkRvsGr3jJD3zk,5582
|
|
7
|
+
fpsketch-0.1.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
8
|
+
fpsketch-0.1.0.dist-info/licenses/LICENSE,sha256=iB2nlbvxGiK09qYiZjKV9iPHSsE_AWepEMRx14Og8Ow,1082
|
|
9
|
+
fpsketch-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 the fpsketch contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|