sillstone 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sillstone-0.2.0/PKG-INFO +13 -0
- sillstone-0.2.0/pyproject.toml +21 -0
- sillstone-0.2.0/setup.cfg +4 -0
- sillstone-0.2.0/sillstone/__init__.py +551 -0
- sillstone-0.2.0/sillstone/_generated.py +238 -0
- sillstone-0.2.0/sillstone.egg-info/PKG-INFO +13 -0
- sillstone-0.2.0/sillstone.egg-info/SOURCES.txt +9 -0
- sillstone-0.2.0/sillstone.egg-info/dependency_links.txt +1 -0
- sillstone-0.2.0/sillstone.egg-info/top_level.txt +1 -0
- sillstone-0.2.0/tests/test_checksum.py +63 -0
- sillstone-0.2.0/tests/test_reference_vectors.py +367 -0
sillstone-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sillstone
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Pure-Rust zero-dependency multimedia, hashing, and array computing suite
|
|
5
|
+
Author: Sillstone Contributors
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/sillstone/sillstone
|
|
8
|
+
Project-URL: Repository, https://github.com/sillstone/sillstone
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Requires-Python: >=3.9
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "sillstone"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "Pure-Rust zero-dependency multimedia, hashing, and array computing suite"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = { text = "Apache-2.0" }
|
|
12
|
+
authors = [{ name = "Sillstone Contributors" }]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Programming Language :: Python :: 3",
|
|
15
|
+
"License :: OSI Approved :: Apache Software License",
|
|
16
|
+
"Operating System :: OS Independent",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
[project.urls]
|
|
20
|
+
Homepage = "https://github.com/sillstone/sillstone"
|
|
21
|
+
Repository = "https://github.com/sillstone/sillstone"
|
|
@@ -0,0 +1,551 @@
|
|
|
1
|
+
"""Sillstone Python SDK: Zero-dependency multimedia, hashing, and array computing suite."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ctypes
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
import struct
|
|
9
|
+
|
|
10
|
+
from ._generated import (
|
|
11
|
+
LIB,
|
|
12
|
+
sillstone_version_major_raw,
|
|
13
|
+
sillstone_version_minor_raw,
|
|
14
|
+
sillstone_version_patch_raw,
|
|
15
|
+
sillstone_buffer_checksum_raw,
|
|
16
|
+
sillstone_digest_sha256_raw,
|
|
17
|
+
sillstone_digest_sha3_256_raw,
|
|
18
|
+
sillstone_digest_sha3_512_raw,
|
|
19
|
+
sillstone_digest_blake2b_512_raw,
|
|
20
|
+
sillstone_digest_blake3_raw,
|
|
21
|
+
sillstone_math_dst1_raw,
|
|
22
|
+
sillstone_math_dst2_raw,
|
|
23
|
+
sillstone_math_dst3_raw,
|
|
24
|
+
sillstone_unicode_grapheme_count_raw,
|
|
25
|
+
sillstone_unicode_char_category_raw,
|
|
26
|
+
sillstone_image_phash_raw,
|
|
27
|
+
sillstone_image_ahash_raw,
|
|
28
|
+
sillstone_image_dhash_raw,
|
|
29
|
+
sillstone_image_whash_raw,
|
|
30
|
+
sillstone_image_colorhash_raw,
|
|
31
|
+
sillstone_text_tlsh_raw,
|
|
32
|
+
sillstone_text_tlsh_distance_raw,
|
|
33
|
+
sillstone_curator_detect_raw,
|
|
34
|
+
sillstone_curator_content_hash_raw,
|
|
35
|
+
sillstone_curator_signature_raw,
|
|
36
|
+
sillstone_curator_describe_raw,
|
|
37
|
+
sillstone_curator_match_raw,
|
|
38
|
+
sillstone_curator_free_raw,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
__version__ = "0.2.0b1"
|
|
42
|
+
|
|
43
|
+
__all__ = [
|
|
44
|
+
"version",
|
|
45
|
+
"buffer_checksum",
|
|
46
|
+
"sha256",
|
|
47
|
+
"sha3_256",
|
|
48
|
+
"sha3_512",
|
|
49
|
+
"blake2b_512",
|
|
50
|
+
"blake3",
|
|
51
|
+
"dst1",
|
|
52
|
+
"dst2",
|
|
53
|
+
"dst3",
|
|
54
|
+
"grapheme_count",
|
|
55
|
+
"char_category",
|
|
56
|
+
"phash",
|
|
57
|
+
"ahash",
|
|
58
|
+
"dhash",
|
|
59
|
+
"whash",
|
|
60
|
+
"colorhash",
|
|
61
|
+
"tlsh",
|
|
62
|
+
"tlsh_distance",
|
|
63
|
+
"detect",
|
|
64
|
+
"content_hash",
|
|
65
|
+
"signature",
|
|
66
|
+
"describe",
|
|
67
|
+
"match_signatures",
|
|
68
|
+
"parse_signature",
|
|
69
|
+
"parse_describe",
|
|
70
|
+
"fnv1a64",
|
|
71
|
+
"Detection",
|
|
72
|
+
"Signature",
|
|
73
|
+
"Description",
|
|
74
|
+
"MatchResult",
|
|
75
|
+
"FfiError",
|
|
76
|
+
"FORMATS",
|
|
77
|
+
"MODALITIES",
|
|
78
|
+
"MATCH_TAGS",
|
|
79
|
+
"STATUS_OK",
|
|
80
|
+
"STATUS_INVALID",
|
|
81
|
+
"STATUS_REJECTED",
|
|
82
|
+
]
|
|
83
|
+
|
|
84
|
+
#: Status codes
|
|
85
|
+
STATUS_OK = 0
|
|
86
|
+
STATUS_INVALID = -1
|
|
87
|
+
STATUS_REJECTED = -2
|
|
88
|
+
|
|
89
|
+
#: Format names by wire code (declaration order)
|
|
90
|
+
FORMATS = ("png", "jpeg", "bmp", "zip", "mp4", "mp3", "wav", "flac", "pdf", "gif", "unknown")
|
|
91
|
+
|
|
92
|
+
#: Modality names by wire code (declaration order)
|
|
93
|
+
MODALITIES = ("image", "audio", "text", "binary", "video")
|
|
94
|
+
|
|
95
|
+
#: Match-outcome tags by wire code (declaration order)
|
|
96
|
+
MATCH_TAGS = ("image", "audio", "text", "binary", "video")
|
|
97
|
+
|
|
98
|
+
#: Signature-stream tag bytes (declaration order)
|
|
99
|
+
SIGNATURE_TAGS = ("image", "audio", "text", "binary", "video")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class FfiError(Exception):
|
|
103
|
+
"""A non-zero status code came back from the cdylib."""
|
|
104
|
+
|
|
105
|
+
def __init__(self, op: str, status: int) -> None:
|
|
106
|
+
kind = {
|
|
107
|
+
STATUS_INVALID: "invalid argument",
|
|
108
|
+
STATUS_REJECTED: "input rejected",
|
|
109
|
+
}.get(status, "unknown failure")
|
|
110
|
+
super().__init__(f"{op} failed: {kind} (status {status})")
|
|
111
|
+
self.status = status
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@dataclass(frozen=True)
|
|
115
|
+
class Detection:
|
|
116
|
+
"""Container sniffing result."""
|
|
117
|
+
format: str
|
|
118
|
+
modality: str
|
|
119
|
+
pending: bool
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@dataclass(frozen=True)
|
|
123
|
+
class Signature:
|
|
124
|
+
"""A tier-2 signature, re-expressed from the signature stream."""
|
|
125
|
+
modality: str
|
|
126
|
+
phash: int | None = None
|
|
127
|
+
peak_count: int | None = None
|
|
128
|
+
frames: int | None = None
|
|
129
|
+
peaks: tuple[tuple[int, int], ...] | None = None
|
|
130
|
+
peaks_bytes: bytes | None = None
|
|
131
|
+
word_count: int | None = None
|
|
132
|
+
words: tuple[int, ...] | None = None
|
|
133
|
+
chunk_count: int | None = None
|
|
134
|
+
digest_bytes: bytes | None = None
|
|
135
|
+
frame_count: int | None = None
|
|
136
|
+
width: int | None = None
|
|
137
|
+
height: int | None = None
|
|
138
|
+
duration_bits: int | None = None
|
|
139
|
+
frame_hashes: tuple[int, ...] | None = None
|
|
140
|
+
minhash: tuple[int, ...] | None = None
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@dataclass(frozen=True)
|
|
144
|
+
class Description:
|
|
145
|
+
"""Both tiers plus facts, re-expressed from the describe stream."""
|
|
146
|
+
format: str
|
|
147
|
+
modality: str
|
|
148
|
+
tier1: str
|
|
149
|
+
signature: Signature
|
|
150
|
+
facts: dict
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
@dataclass(frozen=True)
|
|
154
|
+
class MatchResult:
|
|
155
|
+
"""A match outcome, re-expressed from the slots."""
|
|
156
|
+
tag: str
|
|
157
|
+
slots: tuple[int, ...]
|
|
158
|
+
matched: bool
|
|
159
|
+
|
|
160
|
+
def slot_float(self, index: int) -> float:
|
|
161
|
+
"""Reinterprets slot `index` as an IEEE-754 double."""
|
|
162
|
+
return struct.unpack("<d", struct.pack("<Q", self.slots[index]))[0]
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def version() -> str:
|
|
166
|
+
"""Returns library version string."""
|
|
167
|
+
return f"{sillstone_version_major_raw()}.{sillstone_version_minor_raw()}.{sillstone_version_patch_raw()}"
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def buffer_checksum(data: bytes) -> int:
|
|
171
|
+
"""Computes standard 64-bit FNV-1a checksum over input bytes."""
|
|
172
|
+
return int(sillstone_buffer_checksum_raw(data, len(data)))
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def sha256(data: bytes) -> bytes:
|
|
176
|
+
"""Computes 32-byte SHA-256 digest."""
|
|
177
|
+
out = ctypes.create_string_buffer(32)
|
|
178
|
+
rc = sillstone_digest_sha256_raw(data, len(data), out)
|
|
179
|
+
if rc != 0:
|
|
180
|
+
raise FfiError("sha256", rc)
|
|
181
|
+
return out.raw
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def sha3_256(data: bytes) -> bytes:
|
|
185
|
+
"""Computes 32-byte SHA-3-256 digest."""
|
|
186
|
+
out = ctypes.create_string_buffer(32)
|
|
187
|
+
rc = sillstone_digest_sha3_256_raw(data, len(data), out)
|
|
188
|
+
if rc != 0:
|
|
189
|
+
raise FfiError("sha3_256", rc)
|
|
190
|
+
return out.raw
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def sha3_512(data: bytes) -> bytes:
|
|
194
|
+
"""Computes 64-byte SHA-3-512 digest."""
|
|
195
|
+
out = ctypes.create_string_buffer(64)
|
|
196
|
+
rc = sillstone_digest_sha3_512_raw(data, len(data), out)
|
|
197
|
+
if rc != 0:
|
|
198
|
+
raise FfiError("sha3_512", rc)
|
|
199
|
+
return out.raw
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def blake2b_512(data: bytes) -> bytes:
|
|
203
|
+
"""Computes 64-byte unkeyed BLAKE2b digest."""
|
|
204
|
+
out = ctypes.create_string_buffer(64)
|
|
205
|
+
rc = sillstone_digest_blake2b_512_raw(data, len(data), out)
|
|
206
|
+
if rc != 0:
|
|
207
|
+
raise FfiError("blake2b_512", rc)
|
|
208
|
+
return out.raw
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def blake3(data: bytes) -> bytes:
|
|
212
|
+
"""Computes 32-byte BLAKE3 digest."""
|
|
213
|
+
out = ctypes.create_string_buffer(32)
|
|
214
|
+
rc = sillstone_digest_blake3_raw(data, len(data), out)
|
|
215
|
+
if rc != 0:
|
|
216
|
+
raise FfiError("blake3", rc)
|
|
217
|
+
return out.raw
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def dst1(data: list[float]) -> list[float]:
|
|
221
|
+
"""Computes orthonormal DST-I over input float array."""
|
|
222
|
+
c_in = (ctypes.c_double * len(data))(*data)
|
|
223
|
+
c_out = (ctypes.c_double * len(data))()
|
|
224
|
+
rc = sillstone_math_dst1_raw(c_in, len(data), c_out)
|
|
225
|
+
if rc != 0:
|
|
226
|
+
raise FfiError("dst1", rc)
|
|
227
|
+
return list(c_out)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def dst2(data: list[float]) -> list[float]:
|
|
231
|
+
"""Computes orthonormal DST-II over input float array."""
|
|
232
|
+
c_in = (ctypes.c_double * len(data))(*data)
|
|
233
|
+
c_out = (ctypes.c_double * len(data))()
|
|
234
|
+
rc = sillstone_math_dst2_raw(c_in, len(data), c_out)
|
|
235
|
+
if rc != 0:
|
|
236
|
+
raise FfiError("dst2", rc)
|
|
237
|
+
return list(c_out)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def dst3(data: list[float]) -> list[float]:
|
|
241
|
+
"""Computes orthonormal DST-III over input float array."""
|
|
242
|
+
c_in = (ctypes.c_double * len(data))(*data)
|
|
243
|
+
c_out = (ctypes.c_double * len(data))()
|
|
244
|
+
rc = sillstone_math_dst3_raw(c_in, len(data), c_out)
|
|
245
|
+
if rc != 0:
|
|
246
|
+
raise FfiError("dst3", rc)
|
|
247
|
+
return list(c_out)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def grapheme_count(text: str) -> int:
|
|
251
|
+
"""Counts UAX #29 grapheme clusters in a string."""
|
|
252
|
+
raw = text.encode("utf-8")
|
|
253
|
+
count = ctypes.c_size_t()
|
|
254
|
+
rc = sillstone_unicode_grapheme_count_raw(raw, len(raw), ctypes.byref(count))
|
|
255
|
+
if rc != 0:
|
|
256
|
+
raise FfiError("grapheme_count", rc)
|
|
257
|
+
return count.value
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def char_category(char: str) -> int:
|
|
261
|
+
"""Returns general category tag for a single character."""
|
|
262
|
+
cp = ord(char[0])
|
|
263
|
+
cat = ctypes.c_uint8()
|
|
264
|
+
rc = sillstone_unicode_char_category_raw(cp, ctypes.byref(cat))
|
|
265
|
+
if rc != 0:
|
|
266
|
+
raise FfiError("char_category", rc)
|
|
267
|
+
return cat.value
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def phash(data: bytes, width: int, height: int, layout: int = 0) -> int:
|
|
271
|
+
"""Computes 64-bit perceptual hash (pHash) of an image."""
|
|
272
|
+
out = ctypes.c_uint64()
|
|
273
|
+
rc = sillstone_image_phash_raw(data, len(data), width, height, layout, ctypes.byref(out))
|
|
274
|
+
if rc != 0:
|
|
275
|
+
raise FfiError("phash", rc)
|
|
276
|
+
return out.value
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def ahash(data: bytes, width: int, height: int, layout: int = 0) -> int:
|
|
280
|
+
"""Computes 64-bit average hash (aHash) of an image."""
|
|
281
|
+
out = ctypes.c_uint64()
|
|
282
|
+
rc = sillstone_image_ahash_raw(data, len(data), width, height, layout, ctypes.byref(out))
|
|
283
|
+
if rc != 0:
|
|
284
|
+
raise FfiError("ahash", rc)
|
|
285
|
+
return out.value
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def dhash(data: bytes, width: int, height: int, layout: int = 0) -> int:
|
|
289
|
+
"""Computes 64-bit difference hash (dHash) of an image."""
|
|
290
|
+
out = ctypes.c_uint64()
|
|
291
|
+
rc = sillstone_image_dhash_raw(data, len(data), width, height, layout, ctypes.byref(out))
|
|
292
|
+
if rc != 0:
|
|
293
|
+
raise FfiError("dhash", rc)
|
|
294
|
+
return out.value
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def whash(data: bytes, width: int, height: int, layout: int = 0) -> int:
|
|
298
|
+
"""Computes 64-bit wavelet hash (wHash) of an image."""
|
|
299
|
+
out = ctypes.c_uint64()
|
|
300
|
+
rc = sillstone_image_whash_raw(data, len(data), width, height, layout, ctypes.byref(out))
|
|
301
|
+
if rc != 0:
|
|
302
|
+
raise FfiError("whash", rc)
|
|
303
|
+
return out.value
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def colorhash(data: bytes, width: int, height: int, layout: int = 0) -> int:
|
|
307
|
+
"""Computes 64-bit color distribution hash of an image."""
|
|
308
|
+
out = ctypes.c_uint64()
|
|
309
|
+
rc = sillstone_image_colorhash_raw(data, len(data), width, height, layout, ctypes.byref(out))
|
|
310
|
+
if rc != 0:
|
|
311
|
+
raise FfiError("colorhash", rc)
|
|
312
|
+
return out.value
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def tlsh(data: bytes) -> str:
|
|
316
|
+
"""Computes 70-hex-char TrendMicro Locality Sensitive Hash string."""
|
|
317
|
+
out = ctypes.create_string_buffer(72)
|
|
318
|
+
rc = sillstone_text_tlsh_raw(data, len(data), out)
|
|
319
|
+
if rc != 0:
|
|
320
|
+
raise FfiError("tlsh", rc)
|
|
321
|
+
return out.raw.decode("ascii", errors="replace")
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def tlsh_distance(hash_a: str | bytes, hash_b: str | bytes) -> int:
|
|
325
|
+
"""Computes TLSH distance between two 70-hex-char hashes."""
|
|
326
|
+
a_bytes = hash_a.encode("ascii") if isinstance(hash_a, str) else hash_a
|
|
327
|
+
b_bytes = hash_b.encode("ascii") if isinstance(hash_b, str) else hash_b
|
|
328
|
+
out = ctypes.c_uint32()
|
|
329
|
+
rc = sillstone_text_tlsh_distance_raw(a_bytes, b_bytes, ctypes.byref(out))
|
|
330
|
+
if rc != 0:
|
|
331
|
+
raise FfiError("tlsh_distance", rc)
|
|
332
|
+
return out.value
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def fnv1a64(data: bytes) -> int:
|
|
336
|
+
"""The 64-bit FNV-1a fold the reference sub-digests use."""
|
|
337
|
+
h = 0xCBF29CE484222325
|
|
338
|
+
for b in data:
|
|
339
|
+
h ^= b
|
|
340
|
+
h = (h * 0x100000001B3) & 0xFFFFFFFFFFFFFFFF
|
|
341
|
+
return h
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def detect(data: bytes) -> Detection:
|
|
345
|
+
"""Sniffs container and modality without full decode."""
|
|
346
|
+
fmt = ctypes.c_uint32()
|
|
347
|
+
mod = ctypes.c_uint32()
|
|
348
|
+
pending = ctypes.c_uint32()
|
|
349
|
+
rc = sillstone_curator_detect_raw(
|
|
350
|
+
data, len(data), ctypes.byref(fmt), ctypes.byref(mod), ctypes.byref(pending)
|
|
351
|
+
)
|
|
352
|
+
if rc != 0:
|
|
353
|
+
raise FfiError("curator_detect", rc)
|
|
354
|
+
return Detection(
|
|
355
|
+
format=FORMATS[fmt.value],
|
|
356
|
+
modality=MODALITIES[mod.value],
|
|
357
|
+
pending=pending.value == 1,
|
|
358
|
+
)
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def content_hash(data: bytes) -> bytes:
|
|
362
|
+
"""Computes 32-byte tier-1 content digest."""
|
|
363
|
+
buf = (ctypes.c_ubyte * 32)()
|
|
364
|
+
out_len = ctypes.c_size_t()
|
|
365
|
+
rc = sillstone_curator_content_hash_raw(data, len(data), buf, len(buf), ctypes.byref(out_len))
|
|
366
|
+
if rc != 0:
|
|
367
|
+
raise FfiError("curator_content_hash", rc)
|
|
368
|
+
return bytes(buf)
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def _stream_op(raw_fn, data: bytes) -> bytes:
|
|
372
|
+
out = ctypes.c_void_p()
|
|
373
|
+
out_len = ctypes.c_size_t()
|
|
374
|
+
rc = raw_fn(data, len(data), ctypes.byref(out), ctypes.byref(out_len))
|
|
375
|
+
if rc != 0:
|
|
376
|
+
raise FfiError(raw_fn.__name__, rc)
|
|
377
|
+
try:
|
|
378
|
+
return ctypes.string_at(out, out_len.value)
|
|
379
|
+
finally:
|
|
380
|
+
sillstone_curator_free_raw(out, out_len.value)
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def signature(data: bytes) -> Signature:
|
|
384
|
+
"""Computes tier-2 signature stream and parses it into a Signature object."""
|
|
385
|
+
return parse_signature(_stream_op(sillstone_curator_signature_raw, data))
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def describe(data: bytes) -> Description:
|
|
389
|
+
"""Computes both tiers and parses the describe stream."""
|
|
390
|
+
return parse_describe(_stream_op(sillstone_curator_describe_raw, data))
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def match_signatures(a: bytes, b: bytes) -> MatchResult:
|
|
394
|
+
"""Scores two raw inputs across modalities."""
|
|
395
|
+
tag = ctypes.c_uint32()
|
|
396
|
+
slots = (ctypes.c_uint64 * 4)()
|
|
397
|
+
matched = ctypes.c_uint32()
|
|
398
|
+
rc = sillstone_curator_match_raw(
|
|
399
|
+
a, len(a), b, len(b), ctypes.byref(tag), slots, len(slots), ctypes.byref(matched)
|
|
400
|
+
)
|
|
401
|
+
if rc != 0:
|
|
402
|
+
raise FfiError("curator_match", rc)
|
|
403
|
+
return MatchResult(
|
|
404
|
+
tag=MATCH_TAGS[tag.value],
|
|
405
|
+
slots=tuple(int(s) for s in slots),
|
|
406
|
+
matched=matched.value == 1,
|
|
407
|
+
)
|
|
408
|
+
|
|
409
|
+
def parse_signature(raw: bytes) -> Signature:
|
|
410
|
+
"""Re-expresses the canonical signature stream as a Signature."""
|
|
411
|
+
if len(raw) < 1:
|
|
412
|
+
raise ValueError("signature stream is empty")
|
|
413
|
+
tag = raw[0]
|
|
414
|
+
if tag >= len(SIGNATURE_TAGS):
|
|
415
|
+
raise ValueError(f"unknown signature tag {tag}")
|
|
416
|
+
modality = SIGNATURE_TAGS[tag]
|
|
417
|
+
if modality == "image":
|
|
418
|
+
if len(raw) != 9:
|
|
419
|
+
raise ValueError("image stream is not tag + u64")
|
|
420
|
+
return Signature(modality=modality, phash=int.from_bytes(raw[1:9], "big"))
|
|
421
|
+
if modality == "audio":
|
|
422
|
+
if len(raw) < 13:
|
|
423
|
+
raise ValueError("audio stream is shorter than its 13-byte header")
|
|
424
|
+
peak_count = int.from_bytes(raw[1:9], "big")
|
|
425
|
+
frames = int.from_bytes(raw[9:13], "big")
|
|
426
|
+
payload = raw[13:]
|
|
427
|
+
if peak_count != len(payload) or len(payload) % 6 != 0:
|
|
428
|
+
raise ValueError("audio peak count disagrees with the payload")
|
|
429
|
+
peaks = tuple(
|
|
430
|
+
(
|
|
431
|
+
int.from_bytes(payload[i : i + 4], "little"),
|
|
432
|
+
int.from_bytes(payload[i + 4 : i + 6], "little"),
|
|
433
|
+
)
|
|
434
|
+
for i in range(0, len(payload), 6)
|
|
435
|
+
)
|
|
436
|
+
return Signature(
|
|
437
|
+
modality=modality,
|
|
438
|
+
peak_count=peak_count,
|
|
439
|
+
frames=frames,
|
|
440
|
+
peaks=peaks,
|
|
441
|
+
peaks_bytes=payload,
|
|
442
|
+
)
|
|
443
|
+
if modality == "text":
|
|
444
|
+
if len(raw) < 9:
|
|
445
|
+
raise ValueError("text stream is shorter than its 9-byte header")
|
|
446
|
+
word_count = int.from_bytes(raw[1:9], "big")
|
|
447
|
+
payload = raw[9:]
|
|
448
|
+
if word_count != len(payload) or len(payload) % 8 != 0:
|
|
449
|
+
raise ValueError("text word count disagrees with the payload")
|
|
450
|
+
words = tuple(int.from_bytes(payload[i : i + 8], "big") for i in range(0, len(payload), 8))
|
|
451
|
+
return Signature(modality=modality, word_count=word_count, words=words)
|
|
452
|
+
if modality == "binary":
|
|
453
|
+
if len(raw) < 9:
|
|
454
|
+
raise ValueError("binary stream is shorter than its 9-byte header")
|
|
455
|
+
chunk_count = int.from_bytes(raw[1:9], "big")
|
|
456
|
+
payload = raw[9:]
|
|
457
|
+
if chunk_count != len(payload) or len(payload) % 32 != 0:
|
|
458
|
+
raise ValueError("binary chunk count disagrees with the payload")
|
|
459
|
+
return Signature(modality=modality, chunk_count=chunk_count, digest_bytes=payload)
|
|
460
|
+
# video
|
|
461
|
+
if len(raw) < 18:
|
|
462
|
+
raise ValueError("video stream is shorter than its 18-byte header")
|
|
463
|
+
frame_count = int.from_bytes(raw[1:5], "big")
|
|
464
|
+
width = int.from_bytes(raw[5:9], "big")
|
|
465
|
+
height = int.from_bytes(raw[9:13], "big")
|
|
466
|
+
duration_bits = int.from_bytes(raw[13:21], "big")
|
|
467
|
+
rest = raw[21:]
|
|
468
|
+
if frame_count > len(rest) or frame_count % 8 != 0:
|
|
469
|
+
raise ValueError("video frame count disagrees with the payload")
|
|
470
|
+
frames_payload = rest[:frame_count]
|
|
471
|
+
minhash_payload = rest[frame_count:]
|
|
472
|
+
if len(minhash_payload) % 8 != 0:
|
|
473
|
+
raise ValueError("video minhash payload is not whole words")
|
|
474
|
+
return Signature(
|
|
475
|
+
modality=modality,
|
|
476
|
+
frame_count=frame_count,
|
|
477
|
+
width=width,
|
|
478
|
+
height=height,
|
|
479
|
+
duration_bits=duration_bits,
|
|
480
|
+
frame_hashes=tuple(
|
|
481
|
+
int.from_bytes(frames_payload[i : i + 8], "little")
|
|
482
|
+
for i in range(0, len(frames_payload), 8)
|
|
483
|
+
),
|
|
484
|
+
minhash=tuple(
|
|
485
|
+
int.from_bytes(minhash_payload[i : i + 8], "little")
|
|
486
|
+
for i in range(0, len(minhash_payload), 8)
|
|
487
|
+
),
|
|
488
|
+
)
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def parse_describe(raw: bytes) -> Description:
|
|
492
|
+
"""Re-expresses the canonical describe stream as a Description."""
|
|
493
|
+
if len(raw) < 35:
|
|
494
|
+
raise ValueError("describe stream is shorter than its 35-byte header")
|
|
495
|
+
fmt = raw[0]
|
|
496
|
+
mod = raw[1]
|
|
497
|
+
if fmt >= len(FORMATS) or mod >= len(MODALITIES):
|
|
498
|
+
raise ValueError(f"unknown describe codes {fmt}/{mod}")
|
|
499
|
+
tier1 = raw[2:34].hex()
|
|
500
|
+
modality = MODALITIES[mod]
|
|
501
|
+
sig_end = len(raw) - _facts_len(modality)
|
|
502
|
+
sig = parse_signature(raw[34:sig_end])
|
|
503
|
+
facts_raw = raw[sig_end:]
|
|
504
|
+
facts: dict = {}
|
|
505
|
+
if modality == "image":
|
|
506
|
+
_need(facts_raw, 16)
|
|
507
|
+
facts = {"width": _be32(facts_raw, 0), "height": _be32(facts_raw, 4), "keypoints": _be64(facts_raw, 8)}
|
|
508
|
+
elif modality == "audio":
|
|
509
|
+
_need(facts_raw, 18)
|
|
510
|
+
facts = {
|
|
511
|
+
"sample_rate": _be32(facts_raw, 0),
|
|
512
|
+
"channels": int.from_bytes(facts_raw[4:6], "big"),
|
|
513
|
+
"frames": _be32(facts_raw, 6),
|
|
514
|
+
"peaks": _be64(facts_raw, 10),
|
|
515
|
+
}
|
|
516
|
+
elif modality == "text":
|
|
517
|
+
_need(facts_raw, 16)
|
|
518
|
+
facts = {"words": _be64(facts_raw, 0), "canonical_len": _be64(facts_raw, 8)}
|
|
519
|
+
elif modality == "binary":
|
|
520
|
+
_need(facts_raw, 16)
|
|
521
|
+
facts = {"len": _be64(facts_raw, 0), "chunks": _be64(facts_raw, 8)}
|
|
522
|
+
else:
|
|
523
|
+
_need(facts_raw, 24)
|
|
524
|
+
facts = {
|
|
525
|
+
"width": _be32(facts_raw, 0),
|
|
526
|
+
"height": _be32(facts_raw, 4),
|
|
527
|
+
"duration_bits": _be64(facts_raw, 8),
|
|
528
|
+
"frames_sampled": _be64(facts_raw, 16),
|
|
529
|
+
}
|
|
530
|
+
return Description(format=FORMATS[fmt], modality=modality, tier1=tier1, signature=sig, facts=facts)
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def _be32(raw: bytes, off: int) -> int:
|
|
534
|
+
return int.from_bytes(raw[off : off + 4], "big")
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
def _be64(raw: bytes, off: int) -> int:
|
|
538
|
+
return int.from_bytes(raw[off : off + 8], "big")
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
def _need(raw: bytes, n: int) -> None:
|
|
542
|
+
if len(raw) != n:
|
|
543
|
+
raise ValueError(f"facts block is {len(raw)} bytes, want {n}")
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def _facts_len(modality: str) -> int:
|
|
547
|
+
if modality == "audio":
|
|
548
|
+
return 18
|
|
549
|
+
if modality == "video":
|
|
550
|
+
return 24
|
|
551
|
+
return 16
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
# Generated by sillstone-idl-gen. DO NOT EDIT.
|
|
2
|
+
import ctypes
|
|
3
|
+
import os
|
|
4
|
+
import sys
|
|
5
|
+
|
|
6
|
+
def _load_lib():
|
|
7
|
+
ext = '.dll' if sys.platform == 'win32' else ('.dylib' if sys.platform == 'darwin' else '.so')
|
|
8
|
+
prefix = '' if sys.platform == 'win32' else 'lib'
|
|
9
|
+
lib_name = f'{prefix}sillstone_ffi{ext}'
|
|
10
|
+
dir_path = os.path.dirname(os.path.abspath(__file__))
|
|
11
|
+
candidates = [
|
|
12
|
+
os.path.join(dir_path, lib_name),
|
|
13
|
+
os.path.join(dir_path, '..', '..', '..', 'target', 'release', lib_name),
|
|
14
|
+
os.path.join(dir_path, '..', '..', '..', 'target', 'debug', lib_name),
|
|
15
|
+
]
|
|
16
|
+
for c in candidates:
|
|
17
|
+
if os.path.exists(c):
|
|
18
|
+
return ctypes.CDLL(c)
|
|
19
|
+
return None
|
|
20
|
+
|
|
21
|
+
LIB = _load_lib()
|
|
22
|
+
|
|
23
|
+
def sillstone_version_major_raw(*args):
|
|
24
|
+
if LIB is None:
|
|
25
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
26
|
+
fn = getattr(LIB, 'sillstone_version_major')
|
|
27
|
+
fn.restype = ctypes.c_uint32
|
|
28
|
+
fn.argtypes = []
|
|
29
|
+
return fn(*args)
|
|
30
|
+
|
|
31
|
+
def sillstone_version_minor_raw(*args):
|
|
32
|
+
if LIB is None:
|
|
33
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
34
|
+
fn = getattr(LIB, 'sillstone_version_minor')
|
|
35
|
+
fn.restype = ctypes.c_uint32
|
|
36
|
+
fn.argtypes = []
|
|
37
|
+
return fn(*args)
|
|
38
|
+
|
|
39
|
+
def sillstone_version_patch_raw(*args):
|
|
40
|
+
if LIB is None:
|
|
41
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
42
|
+
fn = getattr(LIB, 'sillstone_version_patch')
|
|
43
|
+
fn.restype = ctypes.c_uint32
|
|
44
|
+
fn.argtypes = []
|
|
45
|
+
return fn(*args)
|
|
46
|
+
|
|
47
|
+
def sillstone_buffer_checksum_raw(*args):
|
|
48
|
+
if LIB is None:
|
|
49
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
50
|
+
fn = getattr(LIB, 'sillstone_buffer_checksum')
|
|
51
|
+
fn.restype = ctypes.c_uint64
|
|
52
|
+
fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t]
|
|
53
|
+
return fn(*args)
|
|
54
|
+
|
|
55
|
+
def sillstone_digest_sha256_raw(*args):
|
|
56
|
+
if LIB is None:
|
|
57
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
58
|
+
fn = getattr(LIB, 'sillstone_digest_sha256')
|
|
59
|
+
fn.restype = ctypes.c_int32
|
|
60
|
+
fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p]
|
|
61
|
+
return fn(*args)
|
|
62
|
+
|
|
63
|
+
def sillstone_digest_sha3_256_raw(*args):
|
|
64
|
+
if LIB is None:
|
|
65
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
66
|
+
fn = getattr(LIB, 'sillstone_digest_sha3_256')
|
|
67
|
+
fn.restype = ctypes.c_int32
|
|
68
|
+
fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p]
|
|
69
|
+
return fn(*args)
|
|
70
|
+
|
|
71
|
+
def sillstone_digest_sha3_512_raw(*args):
|
|
72
|
+
if LIB is None:
|
|
73
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
74
|
+
fn = getattr(LIB, 'sillstone_digest_sha3_512')
|
|
75
|
+
fn.restype = ctypes.c_int32
|
|
76
|
+
fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p]
|
|
77
|
+
return fn(*args)
|
|
78
|
+
|
|
79
|
+
def sillstone_digest_blake2b_512_raw(*args):
|
|
80
|
+
if LIB is None:
|
|
81
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
82
|
+
fn = getattr(LIB, 'sillstone_digest_blake2b_512')
|
|
83
|
+
fn.restype = ctypes.c_int32
|
|
84
|
+
fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p]
|
|
85
|
+
return fn(*args)
|
|
86
|
+
|
|
87
|
+
def sillstone_digest_blake3_raw(*args):
|
|
88
|
+
if LIB is None:
|
|
89
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
90
|
+
fn = getattr(LIB, 'sillstone_digest_blake3')
|
|
91
|
+
fn.restype = ctypes.c_int32
|
|
92
|
+
fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p]
|
|
93
|
+
return fn(*args)
|
|
94
|
+
|
|
95
|
+
def sillstone_math_dst1_raw(*args):
|
|
96
|
+
if LIB is None:
|
|
97
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
98
|
+
fn = getattr(LIB, 'sillstone_math_dst1')
|
|
99
|
+
fn.restype = ctypes.c_int32
|
|
100
|
+
fn.argtypes = [ctypes.POINTER(ctypes.c_double), ctypes.c_size_t, ctypes.POINTER(ctypes.c_double)]
|
|
101
|
+
return fn(*args)
|
|
102
|
+
|
|
103
|
+
def sillstone_math_dst2_raw(*args):
|
|
104
|
+
if LIB is None:
|
|
105
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
106
|
+
fn = getattr(LIB, 'sillstone_math_dst2')
|
|
107
|
+
fn.restype = ctypes.c_int32
|
|
108
|
+
fn.argtypes = [ctypes.POINTER(ctypes.c_double), ctypes.c_size_t, ctypes.POINTER(ctypes.c_double)]
|
|
109
|
+
return fn(*args)
|
|
110
|
+
|
|
111
|
+
def sillstone_math_dst3_raw(*args):
|
|
112
|
+
if LIB is None:
|
|
113
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
114
|
+
fn = getattr(LIB, 'sillstone_math_dst3')
|
|
115
|
+
fn.restype = ctypes.c_int32
|
|
116
|
+
fn.argtypes = [ctypes.POINTER(ctypes.c_double), ctypes.c_size_t, ctypes.POINTER(ctypes.c_double)]
|
|
117
|
+
return fn(*args)
|
|
118
|
+
|
|
119
|
+
def sillstone_unicode_grapheme_count_raw(*args):
|
|
120
|
+
if LIB is None:
|
|
121
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
122
|
+
fn = getattr(LIB, 'sillstone_unicode_grapheme_count')
|
|
123
|
+
fn.restype = ctypes.c_int32
|
|
124
|
+
fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_size_t)]
|
|
125
|
+
return fn(*args)
|
|
126
|
+
|
|
127
|
+
def sillstone_unicode_char_category_raw(*args):
|
|
128
|
+
if LIB is None:
|
|
129
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
130
|
+
fn = getattr(LIB, 'sillstone_unicode_char_category')
|
|
131
|
+
fn.restype = ctypes.c_int32
|
|
132
|
+
fn.argtypes = [ctypes.c_uint32, ctypes.c_void_p]
|
|
133
|
+
return fn(*args)
|
|
134
|
+
|
|
135
|
+
def sillstone_curator_detect_raw(*args):
|
|
136
|
+
if LIB is None:
|
|
137
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
138
|
+
fn = getattr(LIB, 'sillstone_curator_detect')
|
|
139
|
+
fn.restype = ctypes.c_int32
|
|
140
|
+
fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_uint32), ctypes.POINTER(ctypes.c_uint32), ctypes.POINTER(ctypes.c_uint32)]
|
|
141
|
+
return fn(*args)
|
|
142
|
+
|
|
143
|
+
def sillstone_curator_content_hash_raw(*args):
|
|
144
|
+
if LIB is None:
|
|
145
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
146
|
+
fn = getattr(LIB, 'sillstone_curator_content_hash')
|
|
147
|
+
fn.restype = ctypes.c_int32
|
|
148
|
+
fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_size_t)]
|
|
149
|
+
return fn(*args)
|
|
150
|
+
|
|
151
|
+
def sillstone_curator_signature_raw(*args):
|
|
152
|
+
if LIB is None:
|
|
153
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
154
|
+
fn = getattr(LIB, 'sillstone_curator_signature')
|
|
155
|
+
fn.restype = ctypes.c_int32
|
|
156
|
+
fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_void_p), ctypes.POINTER(ctypes.c_size_t)]
|
|
157
|
+
return fn(*args)
|
|
158
|
+
|
|
159
|
+
def sillstone_curator_describe_raw(*args):
|
|
160
|
+
if LIB is None:
|
|
161
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
162
|
+
fn = getattr(LIB, 'sillstone_curator_describe')
|
|
163
|
+
fn.restype = ctypes.c_int32
|
|
164
|
+
fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_void_p), ctypes.POINTER(ctypes.c_size_t)]
|
|
165
|
+
return fn(*args)
|
|
166
|
+
|
|
167
|
+
def sillstone_curator_match_raw(*args):
|
|
168
|
+
if LIB is None:
|
|
169
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
170
|
+
fn = getattr(LIB, 'sillstone_curator_match')
|
|
171
|
+
fn.restype = ctypes.c_int32
|
|
172
|
+
fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_uint32), ctypes.POINTER(ctypes.c_uint64), ctypes.c_size_t, ctypes.POINTER(ctypes.c_uint32)]
|
|
173
|
+
return fn(*args)
|
|
174
|
+
|
|
175
|
+
def sillstone_curator_free_raw(*args):
|
|
176
|
+
if LIB is None:
|
|
177
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
178
|
+
fn = getattr(LIB, 'sillstone_curator_free')
|
|
179
|
+
fn.restype = None
|
|
180
|
+
fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t]
|
|
181
|
+
return fn(*args)
|
|
182
|
+
|
|
183
|
+
def sillstone_image_phash_raw(*args):
|
|
184
|
+
if LIB is None:
|
|
185
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
186
|
+
fn = getattr(LIB, 'sillstone_image_phash')
|
|
187
|
+
fn.restype = ctypes.c_int32
|
|
188
|
+
fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32, ctypes.POINTER(ctypes.c_uint64)]
|
|
189
|
+
return fn(*args)
|
|
190
|
+
|
|
191
|
+
def sillstone_image_ahash_raw(*args):
|
|
192
|
+
if LIB is None:
|
|
193
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
194
|
+
fn = getattr(LIB, 'sillstone_image_ahash')
|
|
195
|
+
fn.restype = ctypes.c_int32
|
|
196
|
+
fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32, ctypes.POINTER(ctypes.c_uint64)]
|
|
197
|
+
return fn(*args)
|
|
198
|
+
|
|
199
|
+
def sillstone_image_dhash_raw(*args):
|
|
200
|
+
if LIB is None:
|
|
201
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
202
|
+
fn = getattr(LIB, 'sillstone_image_dhash')
|
|
203
|
+
fn.restype = ctypes.c_int32
|
|
204
|
+
fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32, ctypes.POINTER(ctypes.c_uint64)]
|
|
205
|
+
return fn(*args)
|
|
206
|
+
|
|
207
|
+
def sillstone_image_whash_raw(*args):
|
|
208
|
+
if LIB is None:
|
|
209
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
210
|
+
fn = getattr(LIB, 'sillstone_image_whash')
|
|
211
|
+
fn.restype = ctypes.c_int32
|
|
212
|
+
fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32, ctypes.POINTER(ctypes.c_uint64)]
|
|
213
|
+
return fn(*args)
|
|
214
|
+
|
|
215
|
+
def sillstone_image_colorhash_raw(*args):
|
|
216
|
+
if LIB is None:
|
|
217
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
218
|
+
fn = getattr(LIB, 'sillstone_image_colorhash')
|
|
219
|
+
fn.restype = ctypes.c_int32
|
|
220
|
+
fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32, ctypes.POINTER(ctypes.c_uint64)]
|
|
221
|
+
return fn(*args)
|
|
222
|
+
|
|
223
|
+
def sillstone_text_tlsh_raw(*args):
|
|
224
|
+
if LIB is None:
|
|
225
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
226
|
+
fn = getattr(LIB, 'sillstone_text_tlsh')
|
|
227
|
+
fn.restype = ctypes.c_int32
|
|
228
|
+
fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p]
|
|
229
|
+
return fn(*args)
|
|
230
|
+
|
|
231
|
+
def sillstone_text_tlsh_distance_raw(*args):
|
|
232
|
+
if LIB is None:
|
|
233
|
+
raise RuntimeError('sillstone_ffi library not loaded')
|
|
234
|
+
fn = getattr(LIB, 'sillstone_text_tlsh_distance')
|
|
235
|
+
fn.restype = ctypes.c_int32
|
|
236
|
+
fn.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint32)]
|
|
237
|
+
return fn(*args)
|
|
238
|
+
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sillstone
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Pure-Rust zero-dependency multimedia, hashing, and array computing suite
|
|
5
|
+
Author: Sillstone Contributors
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/sillstone/sillstone
|
|
8
|
+
Project-URL: Repository, https://github.com/sillstone/sillstone
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Requires-Python: >=3.9
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
sillstone
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
import sys
|
|
3
|
+
import os
|
|
4
|
+
|
|
5
|
+
# Add sdk/python to sys.path
|
|
6
|
+
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
|
7
|
+
|
|
8
|
+
import sillstone
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class TestSillstonePython(unittest.TestCase):
|
|
12
|
+
def test_version(self):
|
|
13
|
+
v = sillstone.version()
|
|
14
|
+
self.assertEqual(v, "0.2.0")
|
|
15
|
+
|
|
16
|
+
def test_checksum_sample(self):
|
|
17
|
+
data = b"sillstone pure rust zero dep"
|
|
18
|
+
res = sillstone.buffer_checksum(data)
|
|
19
|
+
# Expected FNV-1a 64-bit checksum
|
|
20
|
+
# Compute reference in pure Python
|
|
21
|
+
h = 0xcbf29ce484222325
|
|
22
|
+
for b in data:
|
|
23
|
+
h ^= b
|
|
24
|
+
h = (h * 0x0100000001b3) & 0xffffffffffffffff
|
|
25
|
+
self.assertEqual(res, h)
|
|
26
|
+
|
|
27
|
+
def test_digests(self):
|
|
28
|
+
import hashlib
|
|
29
|
+
data = b"abc"
|
|
30
|
+
self.assertEqual(sillstone.sha256(data).hex(), hashlib.sha256(data).hexdigest())
|
|
31
|
+
self.assertEqual(sillstone.sha3_256(data).hex(), hashlib.sha3_256(data).hexdigest())
|
|
32
|
+
self.assertEqual(sillstone.sha3_512(data).hex(), hashlib.sha3_512(data).hexdigest())
|
|
33
|
+
self.assertEqual(sillstone.blake2b_512(data).hex(), hashlib.blake2b(data).hexdigest())
|
|
34
|
+
# BLAKE3 "abc" reference
|
|
35
|
+
self.assertEqual(sillstone.blake3(b"").hex(), "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262")
|
|
36
|
+
|
|
37
|
+
def test_dst(self):
|
|
38
|
+
x = [1.0, 2.0, 3.0, 4.0]
|
|
39
|
+
d1 = sillstone.dst1(x)
|
|
40
|
+
self.assertEqual(len(d1), 4)
|
|
41
|
+
# dst1 is self-inverse when orthonormal
|
|
42
|
+
rec1 = sillstone.dst1(d1)
|
|
43
|
+
for a, b in zip(x, rec1):
|
|
44
|
+
self.assertAlmostEqual(a, b, places=10)
|
|
45
|
+
|
|
46
|
+
# dst2 and dst3 are mutually inverse
|
|
47
|
+
d2 = sillstone.dst2(x)
|
|
48
|
+
rec2 = sillstone.dst3(d2)
|
|
49
|
+
for a, b in zip(x, rec2):
|
|
50
|
+
self.assertAlmostEqual(a, b, places=10)
|
|
51
|
+
|
|
52
|
+
def test_unicode(self):
|
|
53
|
+
# Family emoji (4 emoji chars + 3 ZWJ): 1 grapheme cluster
|
|
54
|
+
fam = "👨👩👧👦"
|
|
55
|
+
self.assertEqual(sillstone.grapheme_count(fam), 1)
|
|
56
|
+
self.assertEqual(sillstone.grapheme_count("Hello"), 5)
|
|
57
|
+
self.assertEqual(sillstone.grapheme_count(""), 0)
|
|
58
|
+
cat_a = sillstone.char_category("A")
|
|
59
|
+
self.assertGreaterEqual(cat_a, 0)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
if __name__ == "__main__":
|
|
63
|
+
unittest.main()
|
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
# Copyright (c) 2026 pith-hash / sillstone Contributors
|
|
3
|
+
"""Hex-exact conformance: the committed reference vectors through Python.
|
|
4
|
+
|
|
5
|
+
Every one of the 45 vectors in the repository-root reference.json is replayed
|
|
6
|
+
through Python ctypes and compared against every recorded field:
|
|
7
|
+
- 13 detect codes
|
|
8
|
+
- 14 tier-2 signature fold sets (pHash, audio peaks, text/video minhash,
|
|
9
|
+
cdc chunk digests, frame hashes)
|
|
10
|
+
- 7 tier-1 / tier-2 describe packets
|
|
11
|
+
- 9 match-outcome slot vectors
|
|
12
|
+
- 2 pinned refusal errors (cross-modality, non-AVC video)
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import hashlib
|
|
18
|
+
import json
|
|
19
|
+
import math
|
|
20
|
+
import os
|
|
21
|
+
import struct
|
|
22
|
+
import unittest
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
from sillstone import (
|
|
26
|
+
FORMATS,
|
|
27
|
+
FfiError,
|
|
28
|
+
MatchResult,
|
|
29
|
+
MODALITIES,
|
|
30
|
+
Description,
|
|
31
|
+
Detection,
|
|
32
|
+
Signature,
|
|
33
|
+
content_hash,
|
|
34
|
+
describe,
|
|
35
|
+
detect,
|
|
36
|
+
fnv1a64,
|
|
37
|
+
match_signatures,
|
|
38
|
+
signature,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
42
|
+
REFERENCE = json.loads((REPO_ROOT / "reference.json").read_text(encoding="utf-8"))["vectors"]
|
|
43
|
+
BY_NAME = {v["name"]: v for v in REFERENCE}
|
|
44
|
+
FIXTURES = REPO_ROOT / "tests" / "fixtures"
|
|
45
|
+
|
|
46
|
+
MASK64 = (1 << 64) - 1
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# ---------------------------------------------------------------------
|
|
50
|
+
# The deterministic inline corpus (transcribed from tools/gen-reference)
|
|
51
|
+
# ---------------------------------------------------------------------
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class SplitMix64:
|
|
55
|
+
"""The 64-bit generator behind every inline recipe."""
|
|
56
|
+
|
|
57
|
+
def __init__(self, seed: int) -> None:
|
|
58
|
+
self.s = seed & MASK64
|
|
59
|
+
|
|
60
|
+
def next_u64(self) -> int:
|
|
61
|
+
self.s = (self.s + 0x9E3779B97F4A7C15) & MASK64
|
|
62
|
+
z = self.s
|
|
63
|
+
z = ((z ^ (z >> 30)) * 0xBF58476D1CE4E5B9) & MASK64
|
|
64
|
+
z = ((z ^ (z >> 27)) * 0x94D049BB133111EB) & MASK64
|
|
65
|
+
return z ^ (z >> 31)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def splitmix_bytes(seed: int, length: int) -> bytes:
|
|
69
|
+
"""One next_u64 per byte, low byte kept — the upstream recipe."""
|
|
70
|
+
rng = SplitMix64(seed)
|
|
71
|
+
return bytes(rng.next_u64() & 0xFF for _ in range(length))
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def blob48(seed: int) -> bytes:
|
|
75
|
+
"""The 48 KiB pseudo-file recipe of the upstream facade suite."""
|
|
76
|
+
return splitmix_bytes(seed, 48 * 1024)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
VOCAB = (
|
|
80
|
+
"hash", "image", "audio", "signal", "codec", "block", "frame", "sample", "stream",
|
|
81
|
+
"vector", "median", "kernel", "window", "raster", "pixel", "tone", "hash", "luma",
|
|
82
|
+
"delta", "chunk", "radix", "filter", "table", "edge",
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def prose(seed: int, size: int) -> str:
|
|
87
|
+
"""Seeded vocabulary words, one space apart, truncated to `size`."""
|
|
88
|
+
rng = SplitMix64(seed)
|
|
89
|
+
out = ""
|
|
90
|
+
while len(out) < size:
|
|
91
|
+
out += VOCAB[rng.next_u64() % len(VOCAB)] + " "
|
|
92
|
+
return out[:size]
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def patterned_pixels(seed: int, w: int, h: int) -> bytes:
|
|
96
|
+
"""Smooth periodic RGB content with real low-frequency energy."""
|
|
97
|
+
rng = SplitMix64(seed)
|
|
98
|
+
fx = float(rng.next_u64() % 5 + 1)
|
|
99
|
+
fy = float(rng.next_u64() % 5 + 1)
|
|
100
|
+
p1 = (rng.next_u64() % 628) / 100.0
|
|
101
|
+
p2 = (rng.next_u64() % 628) / 100.0
|
|
102
|
+
px = bytearray()
|
|
103
|
+
for y in range(h):
|
|
104
|
+
for x in range(w):
|
|
105
|
+
u = x / w
|
|
106
|
+
v = y / h
|
|
107
|
+
r = 127.5 + 120.0 * math.sin(u * fx * (2 * math.pi) + p1)
|
|
108
|
+
g = 127.5 + 120.0 * math.sin(v * fy * (2 * math.pi) + p2)
|
|
109
|
+
b = 127.5 + 120.0 * math.sin((u + v) * (fx + fy) + p1 + p2)
|
|
110
|
+
px += bytes((int(r) & 0xFF, int(g) & 0xFF, int(b) & 0xFF))
|
|
111
|
+
return bytes(px)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def bmp24(w: int, h: int, rgb: bytes) -> bytes:
|
|
115
|
+
"""Bottom-up 24-bit BMP — the suite's deterministic fixture encoder."""
|
|
116
|
+
assert len(rgb) == w * h * 3
|
|
117
|
+
row = ((w * 3 + 3) // 4) * 4
|
|
118
|
+
img = row * h
|
|
119
|
+
f = bytearray()
|
|
120
|
+
f += b"BM"
|
|
121
|
+
f += struct.pack("<I", 54 + img)
|
|
122
|
+
f += b"\x00\x00\x00\x00"
|
|
123
|
+
f += struct.pack("<I", 54)
|
|
124
|
+
f += struct.pack("<I", 40)
|
|
125
|
+
f += struct.pack("<ii", w, h)
|
|
126
|
+
f += struct.pack("<hh", 1, 24)
|
|
127
|
+
f += b"\x00" * 24
|
|
128
|
+
for y in range(h - 1, -1, -1):
|
|
129
|
+
for x in range(w):
|
|
130
|
+
i = (y * w + x) * 3
|
|
131
|
+
f += bytes((rgb[i + 2], rgb[i + 1], rgb[i]))
|
|
132
|
+
f += b"\x00" * (row - w * 3)
|
|
133
|
+
return bytes(f)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def fixture(name: str) -> bytes:
|
|
137
|
+
p = FIXTURES / name
|
|
138
|
+
if not p.is_file():
|
|
139
|
+
raise FileNotFoundError(f"Fixture {p} is missing")
|
|
140
|
+
return p.read_bytes()
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
# Pinned payloads constructed identically to tools/gen-reference:
|
|
144
|
+
PNG48 = fixture("phash_rgb8_48x40.png")
|
|
145
|
+
GRAY40 = fixture("phash_gray8_40x32.png")
|
|
146
|
+
PNG_BASE = fixture("base_444.png")
|
|
147
|
+
JPG_BASE = fixture("base_444.jpg")
|
|
148
|
+
WAV = fixture("tone.wav")
|
|
149
|
+
FLAC = fixture("tone.flac")
|
|
150
|
+
MP3 = fixture("l3_short.mp3")
|
|
151
|
+
MP4 = fixture("a_64x48.mp4")
|
|
152
|
+
MOV = fixture("a_64x48.mov")
|
|
153
|
+
MP4V = fixture("e_mp4v.mp4")
|
|
154
|
+
PDF = fixture("text_page.pdf")
|
|
155
|
+
|
|
156
|
+
BMP_SYNTH = bmp24(48, 40, patterned_pixels(0xBEEF_0001, 48, 40))
|
|
157
|
+
BLOB = blob48(0xD15E_5EED)
|
|
158
|
+
BLOB_SHIFTED = b"\xaa" + blob48(0xD15E_5EED)
|
|
159
|
+
BLOB_OTHER = blob48(0xBADC_0FFE)
|
|
160
|
+
PROSE_A = prose(0xD1, 4096).encode()
|
|
161
|
+
PROSE_B = prose(0xD2, 4096).encode()
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def sha256_hex(data: bytes) -> str:
|
|
165
|
+
return hashlib.sha256(data).hexdigest()
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def le_words(words) -> bytes:
|
|
169
|
+
"""The canonical word fold: little-endian words concatenated."""
|
|
170
|
+
return b"".join(struct.pack("<Q", w & MASK64) for w in words)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
MATCH_INPUTS = {
|
|
174
|
+
"match-image-self": (PNG48, PNG48, "image"),
|
|
175
|
+
"match-image-distinct": (PNG48, GRAY40, "image"),
|
|
176
|
+
"match-image-png-vs-jpeg-same-pixels": (PNG_BASE, JPG_BASE, "image"),
|
|
177
|
+
"match-audio-self": (WAV, WAV, "audio"),
|
|
178
|
+
"match-text-self": (PROSE_A, PROSE_A, "text"),
|
|
179
|
+
"match-text-distinct": (PROSE_A, PROSE_B, "text"),
|
|
180
|
+
"match-binary-near-duplicate": (BLOB, BLOB_SHIFTED, "binary"),
|
|
181
|
+
"match-binary-unrelated": (BLOB, BLOB_OTHER, "binary"),
|
|
182
|
+
"match-video-mp4-vs-mov": (MP4, MOV, "video"),
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
class TestReferenceVectors(unittest.TestCase):
|
|
187
|
+
def test_detect_vectors(self):
|
|
188
|
+
cases = [
|
|
189
|
+
("detect-png", PNG48),
|
|
190
|
+
("detect-jpeg", JPG_BASE),
|
|
191
|
+
("detect-bmp", BMP_SYNTH),
|
|
192
|
+
("detect-wav", WAV),
|
|
193
|
+
("detect-flac", FLAC),
|
|
194
|
+
("detect-mp3", MP3),
|
|
195
|
+
("detect-mp4", MP4),
|
|
196
|
+
("detect-mov", MOV),
|
|
197
|
+
("detect-pdf", PDF),
|
|
198
|
+
("detect-zip", b"PK\x03\x04\x14\x00\x00\x00\x00\x00"),
|
|
199
|
+
("detect-gif", b"GIF89a\x01\x00\x01\x00"),
|
|
200
|
+
("detect-text", b"just some words"),
|
|
201
|
+
("detect-binary", b"\x00\xff\x02\xfe"),
|
|
202
|
+
]
|
|
203
|
+
for name, data in cases:
|
|
204
|
+
with self.subTest(vector=name):
|
|
205
|
+
rec = BY_NAME[name]
|
|
206
|
+
d: Detection = detect(data)
|
|
207
|
+
self.assertEqual(d.format, rec["format"])
|
|
208
|
+
self.assertEqual(d.modality, rec["modality"])
|
|
209
|
+
self.assertEqual(d.pending, rec["pending"])
|
|
210
|
+
self.assertIn(FORMATS.index(d.format), range(len(FORMATS)))
|
|
211
|
+
self.assertIn(MODALITIES.index(d.modality), range(len(MODALITIES)))
|
|
212
|
+
|
|
213
|
+
def assert_signature_matches(self, sig: Signature, rec: dict):
|
|
214
|
+
self.assertEqual(sig.modality, rec["modality"])
|
|
215
|
+
if sig.modality == "image":
|
|
216
|
+
self.assertEqual(sig.phash, int(rec["phash"], 16))
|
|
217
|
+
elif sig.modality == "audio":
|
|
218
|
+
self.assertEqual(sig.peak_count, rec["peak_count"])
|
|
219
|
+
self.assertEqual(sig.frames, rec["frames"])
|
|
220
|
+
self.assertEqual(fnv1a64(sig.peaks_bytes), int(rec["peaks_fnv1a64"], 16))
|
|
221
|
+
self.assertEqual(sha256_hex(sig.peaks_bytes), rec["peaks_sha256"])
|
|
222
|
+
elif sig.modality == "text":
|
|
223
|
+
self.assertEqual(sig.word_count, rec["minhash_words"])
|
|
224
|
+
fold = le_words(sig.words)
|
|
225
|
+
self.assertEqual(len(fold), sig.word_count)
|
|
226
|
+
self.assertEqual(fnv1a64(fold), int(rec["minhash_fnv1a64"], 16))
|
|
227
|
+
elif sig.modality == "binary":
|
|
228
|
+
self.assertEqual(sig.chunk_count, rec["chunk_count"])
|
|
229
|
+
self.assertEqual(fnv1a64(sig.digest_bytes), int(rec["chunks_fnv1a64"], 16))
|
|
230
|
+
self.assertEqual(sha256_hex(sig.digest_bytes), rec["chunks_sha256"])
|
|
231
|
+
else: # video
|
|
232
|
+
self.assertEqual(sig.frame_count, rec["frame_count"])
|
|
233
|
+
frames_fold = le_words(sig.frame_hashes)
|
|
234
|
+
self.assertEqual(len(frames_fold), sig.frame_count)
|
|
235
|
+
self.assertEqual(fnv1a64(frames_fold), int(rec["frames_fnv1a64"], 16))
|
|
236
|
+
self.assertEqual(sha256_hex(frames_fold), rec["frames_sha256"])
|
|
237
|
+
self.assertIsNotNone(sig.minhash)
|
|
238
|
+
minhash_fold = le_words(sig.minhash)
|
|
239
|
+
self.assertEqual(len(minhash_fold), rec["minhash_words"])
|
|
240
|
+
self.assertEqual(fnv1a64(minhash_fold), int(rec["minhash_fnv1a64"], 16))
|
|
241
|
+
self.assertEqual(sig.width, rec["width"])
|
|
242
|
+
self.assertEqual(sig.height, rec["height"])
|
|
243
|
+
self.assertEqual(f"{sig.duration_bits:016x}", rec["duration_bits"])
|
|
244
|
+
|
|
245
|
+
def test_signature_vectors(self):
|
|
246
|
+
cases = [
|
|
247
|
+
("sig-image-phash_rgb8_48x40", PNG48),
|
|
248
|
+
("sig-image-phash_gray8_40x32", GRAY40),
|
|
249
|
+
("sig-image-base_444-png", PNG_BASE),
|
|
250
|
+
("sig-image-base_444-jpeg", JPG_BASE),
|
|
251
|
+
("sig-image-bmp-patterned-48x40", BMP_SYNTH),
|
|
252
|
+
("sig-audio-tone-wav", WAV),
|
|
253
|
+
("sig-audio-tone-flac", FLAC),
|
|
254
|
+
("sig-audio-l3_short-mp3", MP3),
|
|
255
|
+
("sig-text-prose-4kib", PROSE_A),
|
|
256
|
+
("sig-binary-splitmix64-48kib", BLOB),
|
|
257
|
+
("sig-binary-splitmix64-48kib-prefix-insert", BLOB_SHIFTED),
|
|
258
|
+
("sig-binary-splitmix64-48kib-unrelated", BLOB_OTHER),
|
|
259
|
+
("sig-video-a_64x48-mp4", MP4),
|
|
260
|
+
("sig-pdf-text_page", PDF),
|
|
261
|
+
]
|
|
262
|
+
for name, data in cases:
|
|
263
|
+
with self.subTest(vector=name):
|
|
264
|
+
self.assert_signature_matches(signature(data), BY_NAME[name])
|
|
265
|
+
|
|
266
|
+
def test_describe_vectors(self):
|
|
267
|
+
cases = [
|
|
268
|
+
("describe-image-phash_rgb8_48x40", PNG48),
|
|
269
|
+
("describe-image-bmp-patterned-48x40", BMP_SYNTH),
|
|
270
|
+
("describe-audio-tone-wav", WAV),
|
|
271
|
+
("describe-text-prose-4kib", PROSE_A),
|
|
272
|
+
("describe-binary-splitmix64-48kib", BLOB),
|
|
273
|
+
("describe-video-a_64x48-mp4", MP4),
|
|
274
|
+
("describe-pdf-text_page", PDF),
|
|
275
|
+
]
|
|
276
|
+
for name, data in cases:
|
|
277
|
+
with self.subTest(vector=name):
|
|
278
|
+
rec = BY_NAME[name]
|
|
279
|
+
d: Description = describe(data)
|
|
280
|
+
self.assertEqual(d.format, rec["format"])
|
|
281
|
+
self.assertEqual(d.modality, rec["modality"])
|
|
282
|
+
self.assertEqual(d.tier1, rec["tier1"])
|
|
283
|
+
self.assert_signature_matches(d.signature, rec)
|
|
284
|
+
facts = d.facts
|
|
285
|
+
if d.modality == "image":
|
|
286
|
+
self.assertEqual(facts["width"], rec["facts"]["width"])
|
|
287
|
+
self.assertEqual(facts["height"], rec["facts"]["height"])
|
|
288
|
+
self.assertEqual(facts["keypoints"], rec["facts"]["keypoints"])
|
|
289
|
+
elif d.modality == "audio":
|
|
290
|
+
self.assertEqual(facts["sample_rate"], rec["facts"]["sample_rate"])
|
|
291
|
+
self.assertEqual(facts["channels"], rec["facts"]["channels"])
|
|
292
|
+
self.assertEqual(facts["frames"], rec["facts"]["frames"])
|
|
293
|
+
self.assertEqual(facts["peaks"], rec["facts"]["peaks"])
|
|
294
|
+
elif d.modality == "text":
|
|
295
|
+
self.assertEqual(facts["words"], rec["facts"]["words"])
|
|
296
|
+
self.assertEqual(facts["canonical_len"], rec["facts"]["canonical_len"])
|
|
297
|
+
elif d.modality == "binary":
|
|
298
|
+
self.assertEqual(facts["len"], rec["facts"]["len"])
|
|
299
|
+
self.assertEqual(facts["chunks"], rec["facts"]["chunks"])
|
|
300
|
+
else: # video
|
|
301
|
+
self.assertEqual(facts["width"], rec["facts"]["width"])
|
|
302
|
+
self.assertEqual(facts["height"], rec["facts"]["height"])
|
|
303
|
+
self.assertEqual(f"{facts['duration_bits']:016x}", rec["facts"]["duration_bits"])
|
|
304
|
+
self.assertEqual(facts["frames_sampled"], rec["facts"]["frames_sampled"])
|
|
305
|
+
|
|
306
|
+
def test_match_vectors(self):
|
|
307
|
+
for name in sorted(MATCH_INPUTS):
|
|
308
|
+
with self.subTest(vector=name):
|
|
309
|
+
a, b, kind = MATCH_INPUTS[name]
|
|
310
|
+
rec = BY_NAME[name]
|
|
311
|
+
r: MatchResult = match_signatures(a, b)
|
|
312
|
+
self.assertEqual(r.tag, kind)
|
|
313
|
+
self.assertEqual(r.matched, rec["matched"])
|
|
314
|
+
if kind == "image":
|
|
315
|
+
self.assertEqual(r.slots[0], rec["hamming"])
|
|
316
|
+
elif kind == "audio":
|
|
317
|
+
if "votes" in rec and rec["votes"]:
|
|
318
|
+
self.assertEqual(r.slots[0], rec["votes"])
|
|
319
|
+
self.assertEqual(r.slots[1], rec["delta_t"] & MASK64)
|
|
320
|
+
else:
|
|
321
|
+
self.assertEqual(r.slots[0], 0)
|
|
322
|
+
elif kind == "video":
|
|
323
|
+
self.assertEqual(f"{r.slots[0]:016x}", rec["score_bits"])
|
|
324
|
+
self.assertEqual(f"{r.slots[1]:016x}", rec["minhash_jaccard_bits"])
|
|
325
|
+
else:
|
|
326
|
+
self.assertEqual(f"{r.slots[0]:016x}", rec["jaccard_bits"])
|
|
327
|
+
|
|
328
|
+
def test_match_cross_modality_is_pinned_refusal(self):
|
|
329
|
+
rec = BY_NAME["match-error-cross-modality"]
|
|
330
|
+
with self.assertRaises(FfiError) as ctx:
|
|
331
|
+
match_signatures(PNG48, PROSE_A)
|
|
332
|
+
self.assertEqual(ctx.exception.status, -2)
|
|
333
|
+
|
|
334
|
+
def test_error_video_non_avc_is_pinned_refusal(self):
|
|
335
|
+
rec = BY_NAME["error-video-non-avc"]
|
|
336
|
+
with self.assertRaises(FfiError) as ctx:
|
|
337
|
+
signature(MP4V)
|
|
338
|
+
self.assertEqual(ctx.exception.status, -2)
|
|
339
|
+
|
|
340
|
+
def test_corrupt_image_is_rejected(self):
|
|
341
|
+
corrupt = b"\x89PNG\r\n\x1a\ngarbage"
|
|
342
|
+
with self.assertRaises(FfiError) as ctx:
|
|
343
|
+
signature(corrupt)
|
|
344
|
+
self.assertEqual(ctx.exception.status, -2)
|
|
345
|
+
|
|
346
|
+
with self.assertRaises(FfiError) as ctx:
|
|
347
|
+
describe(corrupt)
|
|
348
|
+
self.assertEqual(ctx.exception.status, -2)
|
|
349
|
+
|
|
350
|
+
with self.assertRaises(FfiError) as ctx:
|
|
351
|
+
content_hash(corrupt)
|
|
352
|
+
self.assertEqual(ctx.exception.status, -2)
|
|
353
|
+
|
|
354
|
+
def test_malformed_pdf_is_rejected(self):
|
|
355
|
+
with self.assertRaises(FfiError) as ctx:
|
|
356
|
+
signature(b"%PDF-1.4 garbage not a pdf")
|
|
357
|
+
self.assertEqual(ctx.exception.status, -2)
|
|
358
|
+
|
|
359
|
+
def test_content_hash_literal_pin(self):
|
|
360
|
+
self.assertEqual(
|
|
361
|
+
content_hash(PNG48).hex(),
|
|
362
|
+
"1f7637f88725b1f3c1b60fb5559486349298b2a5525bac5fb1df196651e0cb74",
|
|
363
|
+
)
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
if __name__ == "__main__":
|
|
367
|
+
unittest.main()
|