sillstone 0.2.0b1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,13 @@
1
+ Metadata-Version: 2.4
2
+ Name: sillstone
3
+ Version: 0.2.0b1
4
+ Summary: Pure-Rust zero-dependency multimedia, hashing, and array computing suite
5
+ Author: Sillstone Contributors
6
+ License: Apache-2.0
7
+ Project-URL: Homepage, https://github.com/sillstone/sillstone
8
+ Project-URL: Repository, https://github.com/sillstone/sillstone
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: License :: OSI Approved :: Apache Software License
11
+ Classifier: Operating System :: OS Independent
12
+ Requires-Python: >=3.9
13
+ Description-Content-Type: text/markdown
@@ -0,0 +1,21 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61.0"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "sillstone"
7
+ version = "0.2.0b1"
8
+ description = "Pure-Rust zero-dependency multimedia, hashing, and array computing suite"
9
+ readme = "README.md"
10
+ requires-python = ">=3.9"
11
+ license = { text = "Apache-2.0" }
12
+ authors = [{ name = "Sillstone Contributors" }]
13
+ classifiers = [
14
+ "Programming Language :: Python :: 3",
15
+ "License :: OSI Approved :: Apache Software License",
16
+ "Operating System :: OS Independent",
17
+ ]
18
+
19
+ [project.urls]
20
+ Homepage = "https://github.com/sillstone/sillstone"
21
+ Repository = "https://github.com/sillstone/sillstone"
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,551 @@
1
+ """Sillstone Python SDK: Zero-dependency multimedia, hashing, and array computing suite."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ctypes
6
+ from dataclasses import dataclass
7
+ from pathlib import Path
8
+ import struct
9
+
10
+ from ._generated import (
11
+ LIB,
12
+ sillstone_version_major_raw,
13
+ sillstone_version_minor_raw,
14
+ sillstone_version_patch_raw,
15
+ sillstone_buffer_checksum_raw,
16
+ sillstone_digest_sha256_raw,
17
+ sillstone_digest_sha3_256_raw,
18
+ sillstone_digest_sha3_512_raw,
19
+ sillstone_digest_blake2b_512_raw,
20
+ sillstone_digest_blake3_raw,
21
+ sillstone_math_dst1_raw,
22
+ sillstone_math_dst2_raw,
23
+ sillstone_math_dst3_raw,
24
+ sillstone_unicode_grapheme_count_raw,
25
+ sillstone_unicode_char_category_raw,
26
+ sillstone_image_phash_raw,
27
+ sillstone_image_ahash_raw,
28
+ sillstone_image_dhash_raw,
29
+ sillstone_image_whash_raw,
30
+ sillstone_image_colorhash_raw,
31
+ sillstone_text_tlsh_raw,
32
+ sillstone_text_tlsh_distance_raw,
33
+ sillstone_curator_detect_raw,
34
+ sillstone_curator_content_hash_raw,
35
+ sillstone_curator_signature_raw,
36
+ sillstone_curator_describe_raw,
37
+ sillstone_curator_match_raw,
38
+ sillstone_curator_free_raw,
39
+ )
40
+
41
+ __version__ = "0.2.0b1"
42
+
43
+ __all__ = [
44
+ "version",
45
+ "buffer_checksum",
46
+ "sha256",
47
+ "sha3_256",
48
+ "sha3_512",
49
+ "blake2b_512",
50
+ "blake3",
51
+ "dst1",
52
+ "dst2",
53
+ "dst3",
54
+ "grapheme_count",
55
+ "char_category",
56
+ "phash",
57
+ "ahash",
58
+ "dhash",
59
+ "whash",
60
+ "colorhash",
61
+ "tlsh",
62
+ "tlsh_distance",
63
+ "detect",
64
+ "content_hash",
65
+ "signature",
66
+ "describe",
67
+ "match_signatures",
68
+ "parse_signature",
69
+ "parse_describe",
70
+ "fnv1a64",
71
+ "Detection",
72
+ "Signature",
73
+ "Description",
74
+ "MatchResult",
75
+ "FfiError",
76
+ "FORMATS",
77
+ "MODALITIES",
78
+ "MATCH_TAGS",
79
+ "STATUS_OK",
80
+ "STATUS_INVALID",
81
+ "STATUS_REJECTED",
82
+ ]
83
+
84
+ #: Status codes
85
+ STATUS_OK = 0
86
+ STATUS_INVALID = -1
87
+ STATUS_REJECTED = -2
88
+
89
+ #: Format names by wire code (declaration order)
90
+ FORMATS = ("png", "jpeg", "bmp", "zip", "mp4", "mp3", "wav", "flac", "pdf", "gif", "unknown")
91
+
92
+ #: Modality names by wire code (declaration order)
93
+ MODALITIES = ("image", "audio", "text", "binary", "video")
94
+
95
+ #: Match-outcome tags by wire code (declaration order)
96
+ MATCH_TAGS = ("image", "audio", "text", "binary", "video")
97
+
98
+ #: Signature-stream tag bytes (declaration order)
99
+ SIGNATURE_TAGS = ("image", "audio", "text", "binary", "video")
100
+
101
+
102
+ class FfiError(Exception):
103
+ """A non-zero status code came back from the cdylib."""
104
+
105
+ def __init__(self, op: str, status: int) -> None:
106
+ kind = {
107
+ STATUS_INVALID: "invalid argument",
108
+ STATUS_REJECTED: "input rejected",
109
+ }.get(status, "unknown failure")
110
+ super().__init__(f"{op} failed: {kind} (status {status})")
111
+ self.status = status
112
+
113
+
114
+ @dataclass(frozen=True)
115
+ class Detection:
116
+ """Container sniffing result."""
117
+ format: str
118
+ modality: str
119
+ pending: bool
120
+
121
+
122
+ @dataclass(frozen=True)
123
+ class Signature:
124
+ """A tier-2 signature, re-expressed from the signature stream."""
125
+ modality: str
126
+ phash: int | None = None
127
+ peak_count: int | None = None
128
+ frames: int | None = None
129
+ peaks: tuple[tuple[int, int], ...] | None = None
130
+ peaks_bytes: bytes | None = None
131
+ word_count: int | None = None
132
+ words: tuple[int, ...] | None = None
133
+ chunk_count: int | None = None
134
+ digest_bytes: bytes | None = None
135
+ frame_count: int | None = None
136
+ width: int | None = None
137
+ height: int | None = None
138
+ duration_bits: int | None = None
139
+ frame_hashes: tuple[int, ...] | None = None
140
+ minhash: tuple[int, ...] | None = None
141
+
142
+
143
+ @dataclass(frozen=True)
144
+ class Description:
145
+ """Both tiers plus facts, re-expressed from the describe stream."""
146
+ format: str
147
+ modality: str
148
+ tier1: str
149
+ signature: Signature
150
+ facts: dict
151
+
152
+
153
+ @dataclass(frozen=True)
154
+ class MatchResult:
155
+ """A match outcome, re-expressed from the slots."""
156
+ tag: str
157
+ slots: tuple[int, ...]
158
+ matched: bool
159
+
160
+ def slot_float(self, index: int) -> float:
161
+ """Reinterprets slot `index` as an IEEE-754 double."""
162
+ return struct.unpack("<d", struct.pack("<Q", self.slots[index]))[0]
163
+
164
+
165
+ def version() -> str:
166
+ """Returns library version string."""
167
+ return f"{sillstone_version_major_raw()}.{sillstone_version_minor_raw()}.{sillstone_version_patch_raw()}"
168
+
169
+
170
+ def buffer_checksum(data: bytes) -> int:
171
+ """Computes standard 64-bit FNV-1a checksum over input bytes."""
172
+ return int(sillstone_buffer_checksum_raw(data, len(data)))
173
+
174
+
175
+ def sha256(data: bytes) -> bytes:
176
+ """Computes 32-byte SHA-256 digest."""
177
+ out = ctypes.create_string_buffer(32)
178
+ rc = sillstone_digest_sha256_raw(data, len(data), out)
179
+ if rc != 0:
180
+ raise FfiError("sha256", rc)
181
+ return out.raw
182
+
183
+
184
+ def sha3_256(data: bytes) -> bytes:
185
+ """Computes 32-byte SHA-3-256 digest."""
186
+ out = ctypes.create_string_buffer(32)
187
+ rc = sillstone_digest_sha3_256_raw(data, len(data), out)
188
+ if rc != 0:
189
+ raise FfiError("sha3_256", rc)
190
+ return out.raw
191
+
192
+
193
+ def sha3_512(data: bytes) -> bytes:
194
+ """Computes 64-byte SHA-3-512 digest."""
195
+ out = ctypes.create_string_buffer(64)
196
+ rc = sillstone_digest_sha3_512_raw(data, len(data), out)
197
+ if rc != 0:
198
+ raise FfiError("sha3_512", rc)
199
+ return out.raw
200
+
201
+
202
+ def blake2b_512(data: bytes) -> bytes:
203
+ """Computes 64-byte unkeyed BLAKE2b digest."""
204
+ out = ctypes.create_string_buffer(64)
205
+ rc = sillstone_digest_blake2b_512_raw(data, len(data), out)
206
+ if rc != 0:
207
+ raise FfiError("blake2b_512", rc)
208
+ return out.raw
209
+
210
+
211
+ def blake3(data: bytes) -> bytes:
212
+ """Computes 32-byte BLAKE3 digest."""
213
+ out = ctypes.create_string_buffer(32)
214
+ rc = sillstone_digest_blake3_raw(data, len(data), out)
215
+ if rc != 0:
216
+ raise FfiError("blake3", rc)
217
+ return out.raw
218
+
219
+
220
+ def dst1(data: list[float]) -> list[float]:
221
+ """Computes orthonormal DST-I over input float array."""
222
+ c_in = (ctypes.c_double * len(data))(*data)
223
+ c_out = (ctypes.c_double * len(data))()
224
+ rc = sillstone_math_dst1_raw(c_in, len(data), c_out)
225
+ if rc != 0:
226
+ raise FfiError("dst1", rc)
227
+ return list(c_out)
228
+
229
+
230
+ def dst2(data: list[float]) -> list[float]:
231
+ """Computes orthonormal DST-II over input float array."""
232
+ c_in = (ctypes.c_double * len(data))(*data)
233
+ c_out = (ctypes.c_double * len(data))()
234
+ rc = sillstone_math_dst2_raw(c_in, len(data), c_out)
235
+ if rc != 0:
236
+ raise FfiError("dst2", rc)
237
+ return list(c_out)
238
+
239
+
240
+ def dst3(data: list[float]) -> list[float]:
241
+ """Computes orthonormal DST-III over input float array."""
242
+ c_in = (ctypes.c_double * len(data))(*data)
243
+ c_out = (ctypes.c_double * len(data))()
244
+ rc = sillstone_math_dst3_raw(c_in, len(data), c_out)
245
+ if rc != 0:
246
+ raise FfiError("dst3", rc)
247
+ return list(c_out)
248
+
249
+
250
+ def grapheme_count(text: str) -> int:
251
+ """Counts UAX #29 grapheme clusters in a string."""
252
+ raw = text.encode("utf-8")
253
+ count = ctypes.c_size_t()
254
+ rc = sillstone_unicode_grapheme_count_raw(raw, len(raw), ctypes.byref(count))
255
+ if rc != 0:
256
+ raise FfiError("grapheme_count", rc)
257
+ return count.value
258
+
259
+
260
+ def char_category(char: str) -> int:
261
+ """Returns general category tag for a single character."""
262
+ cp = ord(char[0])
263
+ cat = ctypes.c_uint8()
264
+ rc = sillstone_unicode_char_category_raw(cp, ctypes.byref(cat))
265
+ if rc != 0:
266
+ raise FfiError("char_category", rc)
267
+ return cat.value
268
+
269
+
270
+ def phash(data: bytes, width: int, height: int, layout: int = 0) -> int:
271
+ """Computes 64-bit perceptual hash (pHash) of an image."""
272
+ out = ctypes.c_uint64()
273
+ rc = sillstone_image_phash_raw(data, len(data), width, height, layout, ctypes.byref(out))
274
+ if rc != 0:
275
+ raise FfiError("phash", rc)
276
+ return out.value
277
+
278
+
279
+ def ahash(data: bytes, width: int, height: int, layout: int = 0) -> int:
280
+ """Computes 64-bit average hash (aHash) of an image."""
281
+ out = ctypes.c_uint64()
282
+ rc = sillstone_image_ahash_raw(data, len(data), width, height, layout, ctypes.byref(out))
283
+ if rc != 0:
284
+ raise FfiError("ahash", rc)
285
+ return out.value
286
+
287
+
288
+ def dhash(data: bytes, width: int, height: int, layout: int = 0) -> int:
289
+ """Computes 64-bit difference hash (dHash) of an image."""
290
+ out = ctypes.c_uint64()
291
+ rc = sillstone_image_dhash_raw(data, len(data), width, height, layout, ctypes.byref(out))
292
+ if rc != 0:
293
+ raise FfiError("dhash", rc)
294
+ return out.value
295
+
296
+
297
+ def whash(data: bytes, width: int, height: int, layout: int = 0) -> int:
298
+ """Computes 64-bit wavelet hash (wHash) of an image."""
299
+ out = ctypes.c_uint64()
300
+ rc = sillstone_image_whash_raw(data, len(data), width, height, layout, ctypes.byref(out))
301
+ if rc != 0:
302
+ raise FfiError("whash", rc)
303
+ return out.value
304
+
305
+
306
+ def colorhash(data: bytes, width: int, height: int, layout: int = 0) -> int:
307
+ """Computes 64-bit color distribution hash of an image."""
308
+ out = ctypes.c_uint64()
309
+ rc = sillstone_image_colorhash_raw(data, len(data), width, height, layout, ctypes.byref(out))
310
+ if rc != 0:
311
+ raise FfiError("colorhash", rc)
312
+ return out.value
313
+
314
+
315
+ def tlsh(data: bytes) -> str:
316
+ """Computes 70-hex-char TrendMicro Locality Sensitive Hash string."""
317
+ out = ctypes.create_string_buffer(72)
318
+ rc = sillstone_text_tlsh_raw(data, len(data), out)
319
+ if rc != 0:
320
+ raise FfiError("tlsh", rc)
321
+ return out.raw.decode("ascii", errors="replace")
322
+
323
+
324
+ def tlsh_distance(hash_a: str | bytes, hash_b: str | bytes) -> int:
325
+ """Computes TLSH distance between two 70-hex-char hashes."""
326
+ a_bytes = hash_a.encode("ascii") if isinstance(hash_a, str) else hash_a
327
+ b_bytes = hash_b.encode("ascii") if isinstance(hash_b, str) else hash_b
328
+ out = ctypes.c_uint32()
329
+ rc = sillstone_text_tlsh_distance_raw(a_bytes, b_bytes, ctypes.byref(out))
330
+ if rc != 0:
331
+ raise FfiError("tlsh_distance", rc)
332
+ return out.value
333
+
334
+
335
+ def fnv1a64(data: bytes) -> int:
336
+ """The 64-bit FNV-1a fold the reference sub-digests use."""
337
+ h = 0xCBF29CE484222325
338
+ for b in data:
339
+ h ^= b
340
+ h = (h * 0x100000001B3) & 0xFFFFFFFFFFFFFFFF
341
+ return h
342
+
343
+
344
+ def detect(data: bytes) -> Detection:
345
+ """Sniffs container and modality without full decode."""
346
+ fmt = ctypes.c_uint32()
347
+ mod = ctypes.c_uint32()
348
+ pending = ctypes.c_uint32()
349
+ rc = sillstone_curator_detect_raw(
350
+ data, len(data), ctypes.byref(fmt), ctypes.byref(mod), ctypes.byref(pending)
351
+ )
352
+ if rc != 0:
353
+ raise FfiError("curator_detect", rc)
354
+ return Detection(
355
+ format=FORMATS[fmt.value],
356
+ modality=MODALITIES[mod.value],
357
+ pending=pending.value == 1,
358
+ )
359
+
360
+
361
+ def content_hash(data: bytes) -> bytes:
362
+ """Computes 32-byte tier-1 content digest."""
363
+ buf = (ctypes.c_ubyte * 32)()
364
+ out_len = ctypes.c_size_t()
365
+ rc = sillstone_curator_content_hash_raw(data, len(data), buf, len(buf), ctypes.byref(out_len))
366
+ if rc != 0:
367
+ raise FfiError("curator_content_hash", rc)
368
+ return bytes(buf)
369
+
370
+
371
+ def _stream_op(raw_fn, data: bytes) -> bytes:
372
+ out = ctypes.c_void_p()
373
+ out_len = ctypes.c_size_t()
374
+ rc = raw_fn(data, len(data), ctypes.byref(out), ctypes.byref(out_len))
375
+ if rc != 0:
376
+ raise FfiError(raw_fn.__name__, rc)
377
+ try:
378
+ return ctypes.string_at(out, out_len.value)
379
+ finally:
380
+ sillstone_curator_free_raw(out, out_len.value)
381
+
382
+
383
+ def signature(data: bytes) -> Signature:
384
+ """Computes tier-2 signature stream and parses it into a Signature object."""
385
+ return parse_signature(_stream_op(sillstone_curator_signature_raw, data))
386
+
387
+
388
+ def describe(data: bytes) -> Description:
389
+ """Computes both tiers and parses the describe stream."""
390
+ return parse_describe(_stream_op(sillstone_curator_describe_raw, data))
391
+
392
+
393
+ def match_signatures(a: bytes, b: bytes) -> MatchResult:
394
+ """Scores two raw inputs across modalities."""
395
+ tag = ctypes.c_uint32()
396
+ slots = (ctypes.c_uint64 * 4)()
397
+ matched = ctypes.c_uint32()
398
+ rc = sillstone_curator_match_raw(
399
+ a, len(a), b, len(b), ctypes.byref(tag), slots, len(slots), ctypes.byref(matched)
400
+ )
401
+ if rc != 0:
402
+ raise FfiError("curator_match", rc)
403
+ return MatchResult(
404
+ tag=MATCH_TAGS[tag.value],
405
+ slots=tuple(int(s) for s in slots),
406
+ matched=matched.value == 1,
407
+ )
408
+
409
+ def parse_signature(raw: bytes) -> Signature:
410
+ """Re-expresses the canonical signature stream as a Signature."""
411
+ if len(raw) < 1:
412
+ raise ValueError("signature stream is empty")
413
+ tag = raw[0]
414
+ if tag >= len(SIGNATURE_TAGS):
415
+ raise ValueError(f"unknown signature tag {tag}")
416
+ modality = SIGNATURE_TAGS[tag]
417
+ if modality == "image":
418
+ if len(raw) != 9:
419
+ raise ValueError("image stream is not tag + u64")
420
+ return Signature(modality=modality, phash=int.from_bytes(raw[1:9], "big"))
421
+ if modality == "audio":
422
+ if len(raw) < 13:
423
+ raise ValueError("audio stream is shorter than its 13-byte header")
424
+ peak_count = int.from_bytes(raw[1:9], "big")
425
+ frames = int.from_bytes(raw[9:13], "big")
426
+ payload = raw[13:]
427
+ if peak_count != len(payload) or len(payload) % 6 != 0:
428
+ raise ValueError("audio peak count disagrees with the payload")
429
+ peaks = tuple(
430
+ (
431
+ int.from_bytes(payload[i : i + 4], "little"),
432
+ int.from_bytes(payload[i + 4 : i + 6], "little"),
433
+ )
434
+ for i in range(0, len(payload), 6)
435
+ )
436
+ return Signature(
437
+ modality=modality,
438
+ peak_count=peak_count,
439
+ frames=frames,
440
+ peaks=peaks,
441
+ peaks_bytes=payload,
442
+ )
443
+ if modality == "text":
444
+ if len(raw) < 9:
445
+ raise ValueError("text stream is shorter than its 9-byte header")
446
+ word_count = int.from_bytes(raw[1:9], "big")
447
+ payload = raw[9:]
448
+ if word_count != len(payload) or len(payload) % 8 != 0:
449
+ raise ValueError("text word count disagrees with the payload")
450
+ words = tuple(int.from_bytes(payload[i : i + 8], "big") for i in range(0, len(payload), 8))
451
+ return Signature(modality=modality, word_count=word_count, words=words)
452
+ if modality == "binary":
453
+ if len(raw) < 9:
454
+ raise ValueError("binary stream is shorter than its 9-byte header")
455
+ chunk_count = int.from_bytes(raw[1:9], "big")
456
+ payload = raw[9:]
457
+ if chunk_count != len(payload) or len(payload) % 32 != 0:
458
+ raise ValueError("binary chunk count disagrees with the payload")
459
+ return Signature(modality=modality, chunk_count=chunk_count, digest_bytes=payload)
460
+ # video
461
+ if len(raw) < 18:
462
+ raise ValueError("video stream is shorter than its 18-byte header")
463
+ frame_count = int.from_bytes(raw[1:5], "big")
464
+ width = int.from_bytes(raw[5:9], "big")
465
+ height = int.from_bytes(raw[9:13], "big")
466
+ duration_bits = int.from_bytes(raw[13:21], "big")
467
+ rest = raw[21:]
468
+ if frame_count > len(rest) or frame_count % 8 != 0:
469
+ raise ValueError("video frame count disagrees with the payload")
470
+ frames_payload = rest[:frame_count]
471
+ minhash_payload = rest[frame_count:]
472
+ if len(minhash_payload) % 8 != 0:
473
+ raise ValueError("video minhash payload is not whole words")
474
+ return Signature(
475
+ modality=modality,
476
+ frame_count=frame_count,
477
+ width=width,
478
+ height=height,
479
+ duration_bits=duration_bits,
480
+ frame_hashes=tuple(
481
+ int.from_bytes(frames_payload[i : i + 8], "little")
482
+ for i in range(0, len(frames_payload), 8)
483
+ ),
484
+ minhash=tuple(
485
+ int.from_bytes(minhash_payload[i : i + 8], "little")
486
+ for i in range(0, len(minhash_payload), 8)
487
+ ),
488
+ )
489
+
490
+
491
+ def parse_describe(raw: bytes) -> Description:
492
+ """Re-expresses the canonical describe stream as a Description."""
493
+ if len(raw) < 35:
494
+ raise ValueError("describe stream is shorter than its 35-byte header")
495
+ fmt = raw[0]
496
+ mod = raw[1]
497
+ if fmt >= len(FORMATS) or mod >= len(MODALITIES):
498
+ raise ValueError(f"unknown describe codes {fmt}/{mod}")
499
+ tier1 = raw[2:34].hex()
500
+ modality = MODALITIES[mod]
501
+ sig_end = len(raw) - _facts_len(modality)
502
+ sig = parse_signature(raw[34:sig_end])
503
+ facts_raw = raw[sig_end:]
504
+ facts: dict = {}
505
+ if modality == "image":
506
+ _need(facts_raw, 16)
507
+ facts = {"width": _be32(facts_raw, 0), "height": _be32(facts_raw, 4), "keypoints": _be64(facts_raw, 8)}
508
+ elif modality == "audio":
509
+ _need(facts_raw, 18)
510
+ facts = {
511
+ "sample_rate": _be32(facts_raw, 0),
512
+ "channels": int.from_bytes(facts_raw[4:6], "big"),
513
+ "frames": _be32(facts_raw, 6),
514
+ "peaks": _be64(facts_raw, 10),
515
+ }
516
+ elif modality == "text":
517
+ _need(facts_raw, 16)
518
+ facts = {"words": _be64(facts_raw, 0), "canonical_len": _be64(facts_raw, 8)}
519
+ elif modality == "binary":
520
+ _need(facts_raw, 16)
521
+ facts = {"len": _be64(facts_raw, 0), "chunks": _be64(facts_raw, 8)}
522
+ else:
523
+ _need(facts_raw, 24)
524
+ facts = {
525
+ "width": _be32(facts_raw, 0),
526
+ "height": _be32(facts_raw, 4),
527
+ "duration_bits": _be64(facts_raw, 8),
528
+ "frames_sampled": _be64(facts_raw, 16),
529
+ }
530
+ return Description(format=FORMATS[fmt], modality=modality, tier1=tier1, signature=sig, facts=facts)
531
+
532
+
533
+ def _be32(raw: bytes, off: int) -> int:
534
+ return int.from_bytes(raw[off : off + 4], "big")
535
+
536
+
537
+ def _be64(raw: bytes, off: int) -> int:
538
+ return int.from_bytes(raw[off : off + 8], "big")
539
+
540
+
541
+ def _need(raw: bytes, n: int) -> None:
542
+ if len(raw) != n:
543
+ raise ValueError(f"facts block is {len(raw)} bytes, want {n}")
544
+
545
+
546
+ def _facts_len(modality: str) -> int:
547
+ if modality == "audio":
548
+ return 18
549
+ if modality == "video":
550
+ return 24
551
+ return 16
@@ -0,0 +1,238 @@
1
+ # Generated by sillstone-idl-gen. DO NOT EDIT.
2
+ import ctypes
3
+ import os
4
+ import sys
5
+
6
+ def _load_lib():
7
+ ext = '.dll' if sys.platform == 'win32' else ('.dylib' if sys.platform == 'darwin' else '.so')
8
+ prefix = '' if sys.platform == 'win32' else 'lib'
9
+ lib_name = f'{prefix}sillstone_ffi{ext}'
10
+ dir_path = os.path.dirname(os.path.abspath(__file__))
11
+ candidates = [
12
+ os.path.join(dir_path, lib_name),
13
+ os.path.join(dir_path, '..', '..', '..', 'target', 'release', lib_name),
14
+ os.path.join(dir_path, '..', '..', '..', 'target', 'debug', lib_name),
15
+ ]
16
+ for c in candidates:
17
+ if os.path.exists(c):
18
+ return ctypes.CDLL(c)
19
+ return None
20
+
21
+ LIB = _load_lib()
22
+
23
+ def sillstone_version_major_raw(*args):
24
+ if LIB is None:
25
+ raise RuntimeError('sillstone_ffi library not loaded')
26
+ fn = getattr(LIB, 'sillstone_version_major')
27
+ fn.restype = ctypes.c_uint32
28
+ fn.argtypes = []
29
+ return fn(*args)
30
+
31
+ def sillstone_version_minor_raw(*args):
32
+ if LIB is None:
33
+ raise RuntimeError('sillstone_ffi library not loaded')
34
+ fn = getattr(LIB, 'sillstone_version_minor')
35
+ fn.restype = ctypes.c_uint32
36
+ fn.argtypes = []
37
+ return fn(*args)
38
+
39
+ def sillstone_version_patch_raw(*args):
40
+ if LIB is None:
41
+ raise RuntimeError('sillstone_ffi library not loaded')
42
+ fn = getattr(LIB, 'sillstone_version_patch')
43
+ fn.restype = ctypes.c_uint32
44
+ fn.argtypes = []
45
+ return fn(*args)
46
+
47
+ def sillstone_buffer_checksum_raw(*args):
48
+ if LIB is None:
49
+ raise RuntimeError('sillstone_ffi library not loaded')
50
+ fn = getattr(LIB, 'sillstone_buffer_checksum')
51
+ fn.restype = ctypes.c_uint64
52
+ fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t]
53
+ return fn(*args)
54
+
55
+ def sillstone_digest_sha256_raw(*args):
56
+ if LIB is None:
57
+ raise RuntimeError('sillstone_ffi library not loaded')
58
+ fn = getattr(LIB, 'sillstone_digest_sha256')
59
+ fn.restype = ctypes.c_int32
60
+ fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p]
61
+ return fn(*args)
62
+
63
+ def sillstone_digest_sha3_256_raw(*args):
64
+ if LIB is None:
65
+ raise RuntimeError('sillstone_ffi library not loaded')
66
+ fn = getattr(LIB, 'sillstone_digest_sha3_256')
67
+ fn.restype = ctypes.c_int32
68
+ fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p]
69
+ return fn(*args)
70
+
71
+ def sillstone_digest_sha3_512_raw(*args):
72
+ if LIB is None:
73
+ raise RuntimeError('sillstone_ffi library not loaded')
74
+ fn = getattr(LIB, 'sillstone_digest_sha3_512')
75
+ fn.restype = ctypes.c_int32
76
+ fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p]
77
+ return fn(*args)
78
+
79
+ def sillstone_digest_blake2b_512_raw(*args):
80
+ if LIB is None:
81
+ raise RuntimeError('sillstone_ffi library not loaded')
82
+ fn = getattr(LIB, 'sillstone_digest_blake2b_512')
83
+ fn.restype = ctypes.c_int32
84
+ fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p]
85
+ return fn(*args)
86
+
87
+ def sillstone_digest_blake3_raw(*args):
88
+ if LIB is None:
89
+ raise RuntimeError('sillstone_ffi library not loaded')
90
+ fn = getattr(LIB, 'sillstone_digest_blake3')
91
+ fn.restype = ctypes.c_int32
92
+ fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p]
93
+ return fn(*args)
94
+
95
+ def sillstone_math_dst1_raw(*args):
96
+ if LIB is None:
97
+ raise RuntimeError('sillstone_ffi library not loaded')
98
+ fn = getattr(LIB, 'sillstone_math_dst1')
99
+ fn.restype = ctypes.c_int32
100
+ fn.argtypes = [ctypes.POINTER(ctypes.c_double), ctypes.c_size_t, ctypes.POINTER(ctypes.c_double)]
101
+ return fn(*args)
102
+
103
+ def sillstone_math_dst2_raw(*args):
104
+ if LIB is None:
105
+ raise RuntimeError('sillstone_ffi library not loaded')
106
+ fn = getattr(LIB, 'sillstone_math_dst2')
107
+ fn.restype = ctypes.c_int32
108
+ fn.argtypes = [ctypes.POINTER(ctypes.c_double), ctypes.c_size_t, ctypes.POINTER(ctypes.c_double)]
109
+ return fn(*args)
110
+
111
+ def sillstone_math_dst3_raw(*args):
112
+ if LIB is None:
113
+ raise RuntimeError('sillstone_ffi library not loaded')
114
+ fn = getattr(LIB, 'sillstone_math_dst3')
115
+ fn.restype = ctypes.c_int32
116
+ fn.argtypes = [ctypes.POINTER(ctypes.c_double), ctypes.c_size_t, ctypes.POINTER(ctypes.c_double)]
117
+ return fn(*args)
118
+
119
+ def sillstone_unicode_grapheme_count_raw(*args):
120
+ if LIB is None:
121
+ raise RuntimeError('sillstone_ffi library not loaded')
122
+ fn = getattr(LIB, 'sillstone_unicode_grapheme_count')
123
+ fn.restype = ctypes.c_int32
124
+ fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_size_t)]
125
+ return fn(*args)
126
+
127
+ def sillstone_unicode_char_category_raw(*args):
128
+ if LIB is None:
129
+ raise RuntimeError('sillstone_ffi library not loaded')
130
+ fn = getattr(LIB, 'sillstone_unicode_char_category')
131
+ fn.restype = ctypes.c_int32
132
+ fn.argtypes = [ctypes.c_uint32, ctypes.c_void_p]
133
+ return fn(*args)
134
+
135
+ def sillstone_curator_detect_raw(*args):
136
+ if LIB is None:
137
+ raise RuntimeError('sillstone_ffi library not loaded')
138
+ fn = getattr(LIB, 'sillstone_curator_detect')
139
+ fn.restype = ctypes.c_int32
140
+ fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_uint32), ctypes.POINTER(ctypes.c_uint32), ctypes.POINTER(ctypes.c_uint32)]
141
+ return fn(*args)
142
+
143
+ def sillstone_curator_content_hash_raw(*args):
144
+ if LIB is None:
145
+ raise RuntimeError('sillstone_ffi library not loaded')
146
+ fn = getattr(LIB, 'sillstone_curator_content_hash')
147
+ fn.restype = ctypes.c_int32
148
+ fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_size_t)]
149
+ return fn(*args)
150
+
151
+ def sillstone_curator_signature_raw(*args):
152
+ if LIB is None:
153
+ raise RuntimeError('sillstone_ffi library not loaded')
154
+ fn = getattr(LIB, 'sillstone_curator_signature')
155
+ fn.restype = ctypes.c_int32
156
+ fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_void_p), ctypes.POINTER(ctypes.c_size_t)]
157
+ return fn(*args)
158
+
159
+ def sillstone_curator_describe_raw(*args):
160
+ if LIB is None:
161
+ raise RuntimeError('sillstone_ffi library not loaded')
162
+ fn = getattr(LIB, 'sillstone_curator_describe')
163
+ fn.restype = ctypes.c_int32
164
+ fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_void_p), ctypes.POINTER(ctypes.c_size_t)]
165
+ return fn(*args)
166
+
167
+ def sillstone_curator_match_raw(*args):
168
+ if LIB is None:
169
+ raise RuntimeError('sillstone_ffi library not loaded')
170
+ fn = getattr(LIB, 'sillstone_curator_match')
171
+ fn.restype = ctypes.c_int32
172
+ fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_uint32), ctypes.POINTER(ctypes.c_uint64), ctypes.c_size_t, ctypes.POINTER(ctypes.c_uint32)]
173
+ return fn(*args)
174
+
175
+ def sillstone_curator_free_raw(*args):
176
+ if LIB is None:
177
+ raise RuntimeError('sillstone_ffi library not loaded')
178
+ fn = getattr(LIB, 'sillstone_curator_free')
179
+ fn.restype = None
180
+ fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t]
181
+ return fn(*args)
182
+
183
+ def sillstone_image_phash_raw(*args):
184
+ if LIB is None:
185
+ raise RuntimeError('sillstone_ffi library not loaded')
186
+ fn = getattr(LIB, 'sillstone_image_phash')
187
+ fn.restype = ctypes.c_int32
188
+ fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32, ctypes.POINTER(ctypes.c_uint64)]
189
+ return fn(*args)
190
+
191
+ def sillstone_image_ahash_raw(*args):
192
+ if LIB is None:
193
+ raise RuntimeError('sillstone_ffi library not loaded')
194
+ fn = getattr(LIB, 'sillstone_image_ahash')
195
+ fn.restype = ctypes.c_int32
196
+ fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32, ctypes.POINTER(ctypes.c_uint64)]
197
+ return fn(*args)
198
+
199
+ def sillstone_image_dhash_raw(*args):
200
+ if LIB is None:
201
+ raise RuntimeError('sillstone_ffi library not loaded')
202
+ fn = getattr(LIB, 'sillstone_image_dhash')
203
+ fn.restype = ctypes.c_int32
204
+ fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32, ctypes.POINTER(ctypes.c_uint64)]
205
+ return fn(*args)
206
+
207
+ def sillstone_image_whash_raw(*args):
208
+ if LIB is None:
209
+ raise RuntimeError('sillstone_ffi library not loaded')
210
+ fn = getattr(LIB, 'sillstone_image_whash')
211
+ fn.restype = ctypes.c_int32
212
+ fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32, ctypes.POINTER(ctypes.c_uint64)]
213
+ return fn(*args)
214
+
215
+ def sillstone_image_colorhash_raw(*args):
216
+ if LIB is None:
217
+ raise RuntimeError('sillstone_ffi library not loaded')
218
+ fn = getattr(LIB, 'sillstone_image_colorhash')
219
+ fn.restype = ctypes.c_int32
220
+ fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32, ctypes.POINTER(ctypes.c_uint64)]
221
+ return fn(*args)
222
+
223
+ def sillstone_text_tlsh_raw(*args):
224
+ if LIB is None:
225
+ raise RuntimeError('sillstone_ffi library not loaded')
226
+ fn = getattr(LIB, 'sillstone_text_tlsh')
227
+ fn.restype = ctypes.c_int32
228
+ fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p]
229
+ return fn(*args)
230
+
231
+ def sillstone_text_tlsh_distance_raw(*args):
232
+ if LIB is None:
233
+ raise RuntimeError('sillstone_ffi library not loaded')
234
+ fn = getattr(LIB, 'sillstone_text_tlsh_distance')
235
+ fn.restype = ctypes.c_int32
236
+ fn.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint32)]
237
+ return fn(*args)
238
+
@@ -0,0 +1,13 @@
1
+ Metadata-Version: 2.4
2
+ Name: sillstone
3
+ Version: 0.2.0b1
4
+ Summary: Pure-Rust zero-dependency multimedia, hashing, and array computing suite
5
+ Author: Sillstone Contributors
6
+ License: Apache-2.0
7
+ Project-URL: Homepage, https://github.com/sillstone/sillstone
8
+ Project-URL: Repository, https://github.com/sillstone/sillstone
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: License :: OSI Approved :: Apache Software License
11
+ Classifier: Operating System :: OS Independent
12
+ Requires-Python: >=3.9
13
+ Description-Content-Type: text/markdown
@@ -0,0 +1,9 @@
1
+ pyproject.toml
2
+ sillstone/__init__.py
3
+ sillstone/_generated.py
4
+ sillstone.egg-info/PKG-INFO
5
+ sillstone.egg-info/SOURCES.txt
6
+ sillstone.egg-info/dependency_links.txt
7
+ sillstone.egg-info/top_level.txt
8
+ tests/test_checksum.py
9
+ tests/test_reference_vectors.py
@@ -0,0 +1 @@
1
+ sillstone
@@ -0,0 +1,63 @@
1
+ import unittest
2
+ import sys
3
+ import os
4
+
5
+ # Add sdk/python to sys.path
6
+ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
7
+
8
+ import sillstone
9
+
10
+
11
+ class TestSillstonePython(unittest.TestCase):
12
+ def test_version(self):
13
+ v = sillstone.version()
14
+ self.assertEqual(v, "0.2.0")
15
+
16
+ def test_checksum_sample(self):
17
+ data = b"sillstone pure rust zero dep"
18
+ res = sillstone.buffer_checksum(data)
19
+ # Expected FNV-1a 64-bit checksum
20
+ # Compute reference in pure Python
21
+ h = 0xcbf29ce484222325
22
+ for b in data:
23
+ h ^= b
24
+ h = (h * 0x0100000001b3) & 0xffffffffffffffff
25
+ self.assertEqual(res, h)
26
+
27
+ def test_digests(self):
28
+ import hashlib
29
+ data = b"abc"
30
+ self.assertEqual(sillstone.sha256(data).hex(), hashlib.sha256(data).hexdigest())
31
+ self.assertEqual(sillstone.sha3_256(data).hex(), hashlib.sha3_256(data).hexdigest())
32
+ self.assertEqual(sillstone.sha3_512(data).hex(), hashlib.sha3_512(data).hexdigest())
33
+ self.assertEqual(sillstone.blake2b_512(data).hex(), hashlib.blake2b(data).hexdigest())
34
+ # BLAKE3 "abc" reference
35
+ self.assertEqual(sillstone.blake3(b"").hex(), "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262")
36
+
37
+ def test_dst(self):
38
+ x = [1.0, 2.0, 3.0, 4.0]
39
+ d1 = sillstone.dst1(x)
40
+ self.assertEqual(len(d1), 4)
41
+ # dst1 is self-inverse when orthonormal
42
+ rec1 = sillstone.dst1(d1)
43
+ for a, b in zip(x, rec1):
44
+ self.assertAlmostEqual(a, b, places=10)
45
+
46
+ # dst2 and dst3 are mutually inverse
47
+ d2 = sillstone.dst2(x)
48
+ rec2 = sillstone.dst3(d2)
49
+ for a, b in zip(x, rec2):
50
+ self.assertAlmostEqual(a, b, places=10)
51
+
52
+ def test_unicode(self):
53
+ # Family emoji (4 emoji chars + 3 ZWJ): 1 grapheme cluster
54
+ fam = "👨‍👩‍👧‍👦"
55
+ self.assertEqual(sillstone.grapheme_count(fam), 1)
56
+ self.assertEqual(sillstone.grapheme_count("Hello"), 5)
57
+ self.assertEqual(sillstone.grapheme_count(""), 0)
58
+ cat_a = sillstone.char_category("A")
59
+ self.assertGreaterEqual(cat_a, 0)
60
+
61
+
62
+ if __name__ == "__main__":
63
+ unittest.main()
@@ -0,0 +1,367 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # Copyright (c) 2026 pith-hash / sillstone Contributors
3
+ """Hex-exact conformance: the committed reference vectors through Python.
4
+
5
+ Every one of the 45 vectors in the repository-root reference.json is replayed
6
+ through Python ctypes and compared against every recorded field:
7
+ - 13 detect codes
8
+ - 14 tier-2 signature fold sets (pHash, audio peaks, text/video minhash,
9
+ cdc chunk digests, frame hashes)
10
+ - 7 tier-1 / tier-2 describe packets
11
+ - 9 match-outcome slot vectors
12
+ - 2 pinned refusal errors (cross-modality, non-AVC video)
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import hashlib
18
+ import json
19
+ import math
20
+ import os
21
+ import struct
22
+ import unittest
23
+ from pathlib import Path
24
+
25
+ from sillstone import (
26
+ FORMATS,
27
+ FfiError,
28
+ MatchResult,
29
+ MODALITIES,
30
+ Description,
31
+ Detection,
32
+ Signature,
33
+ content_hash,
34
+ describe,
35
+ detect,
36
+ fnv1a64,
37
+ match_signatures,
38
+ signature,
39
+ )
40
+
41
+ REPO_ROOT = Path(__file__).resolve().parents[3]
42
+ REFERENCE = json.loads((REPO_ROOT / "reference.json").read_text(encoding="utf-8"))["vectors"]
43
+ BY_NAME = {v["name"]: v for v in REFERENCE}
44
+ FIXTURES = REPO_ROOT / "tests" / "fixtures"
45
+
46
+ MASK64 = (1 << 64) - 1
47
+
48
+
49
+ # ---------------------------------------------------------------------
50
+ # The deterministic inline corpus (transcribed from tools/gen-reference)
51
+ # ---------------------------------------------------------------------
52
+
53
+
54
+ class SplitMix64:
55
+ """The 64-bit generator behind every inline recipe."""
56
+
57
+ def __init__(self, seed: int) -> None:
58
+ self.s = seed & MASK64
59
+
60
+ def next_u64(self) -> int:
61
+ self.s = (self.s + 0x9E3779B97F4A7C15) & MASK64
62
+ z = self.s
63
+ z = ((z ^ (z >> 30)) * 0xBF58476D1CE4E5B9) & MASK64
64
+ z = ((z ^ (z >> 27)) * 0x94D049BB133111EB) & MASK64
65
+ return z ^ (z >> 31)
66
+
67
+
68
+ def splitmix_bytes(seed: int, length: int) -> bytes:
69
+ """One next_u64 per byte, low byte kept — the upstream recipe."""
70
+ rng = SplitMix64(seed)
71
+ return bytes(rng.next_u64() & 0xFF for _ in range(length))
72
+
73
+
74
+ def blob48(seed: int) -> bytes:
75
+ """The 48 KiB pseudo-file recipe of the upstream facade suite."""
76
+ return splitmix_bytes(seed, 48 * 1024)
77
+
78
+
79
+ VOCAB = (
80
+ "hash", "image", "audio", "signal", "codec", "block", "frame", "sample", "stream",
81
+ "vector", "median", "kernel", "window", "raster", "pixel", "tone", "hash", "luma",
82
+ "delta", "chunk", "radix", "filter", "table", "edge",
83
+ )
84
+
85
+
86
+ def prose(seed: int, size: int) -> str:
87
+ """Seeded vocabulary words, one space apart, truncated to `size`."""
88
+ rng = SplitMix64(seed)
89
+ out = ""
90
+ while len(out) < size:
91
+ out += VOCAB[rng.next_u64() % len(VOCAB)] + " "
92
+ return out[:size]
93
+
94
+
95
+ def patterned_pixels(seed: int, w: int, h: int) -> bytes:
96
+ """Smooth periodic RGB content with real low-frequency energy."""
97
+ rng = SplitMix64(seed)
98
+ fx = float(rng.next_u64() % 5 + 1)
99
+ fy = float(rng.next_u64() % 5 + 1)
100
+ p1 = (rng.next_u64() % 628) / 100.0
101
+ p2 = (rng.next_u64() % 628) / 100.0
102
+ px = bytearray()
103
+ for y in range(h):
104
+ for x in range(w):
105
+ u = x / w
106
+ v = y / h
107
+ r = 127.5 + 120.0 * math.sin(u * fx * (2 * math.pi) + p1)
108
+ g = 127.5 + 120.0 * math.sin(v * fy * (2 * math.pi) + p2)
109
+ b = 127.5 + 120.0 * math.sin((u + v) * (fx + fy) + p1 + p2)
110
+ px += bytes((int(r) & 0xFF, int(g) & 0xFF, int(b) & 0xFF))
111
+ return bytes(px)
112
+
113
+
114
+ def bmp24(w: int, h: int, rgb: bytes) -> bytes:
115
+ """Bottom-up 24-bit BMP — the suite's deterministic fixture encoder."""
116
+ assert len(rgb) == w * h * 3
117
+ row = ((w * 3 + 3) // 4) * 4
118
+ img = row * h
119
+ f = bytearray()
120
+ f += b"BM"
121
+ f += struct.pack("<I", 54 + img)
122
+ f += b"\x00\x00\x00\x00"
123
+ f += struct.pack("<I", 54)
124
+ f += struct.pack("<I", 40)
125
+ f += struct.pack("<ii", w, h)
126
+ f += struct.pack("<hh", 1, 24)
127
+ f += b"\x00" * 24
128
+ for y in range(h - 1, -1, -1):
129
+ for x in range(w):
130
+ i = (y * w + x) * 3
131
+ f += bytes((rgb[i + 2], rgb[i + 1], rgb[i]))
132
+ f += b"\x00" * (row - w * 3)
133
+ return bytes(f)
134
+
135
+
136
+ def fixture(name: str) -> bytes:
137
+ p = FIXTURES / name
138
+ if not p.is_file():
139
+ raise FileNotFoundError(f"Fixture {p} is missing")
140
+ return p.read_bytes()
141
+
142
+
143
+ # Pinned payloads constructed identically to tools/gen-reference:
144
+ PNG48 = fixture("phash_rgb8_48x40.png")
145
+ GRAY40 = fixture("phash_gray8_40x32.png")
146
+ PNG_BASE = fixture("base_444.png")
147
+ JPG_BASE = fixture("base_444.jpg")
148
+ WAV = fixture("tone.wav")
149
+ FLAC = fixture("tone.flac")
150
+ MP3 = fixture("l3_short.mp3")
151
+ MP4 = fixture("a_64x48.mp4")
152
+ MOV = fixture("a_64x48.mov")
153
+ MP4V = fixture("e_mp4v.mp4")
154
+ PDF = fixture("text_page.pdf")
155
+
156
+ BMP_SYNTH = bmp24(48, 40, patterned_pixels(0xBEEF_0001, 48, 40))
157
+ BLOB = blob48(0xD15E_5EED)
158
+ BLOB_SHIFTED = b"\xaa" + blob48(0xD15E_5EED)
159
+ BLOB_OTHER = blob48(0xBADC_0FFE)
160
+ PROSE_A = prose(0xD1, 4096).encode()
161
+ PROSE_B = prose(0xD2, 4096).encode()
162
+
163
+
164
+ def sha256_hex(data: bytes) -> str:
165
+ return hashlib.sha256(data).hexdigest()
166
+
167
+
168
+ def le_words(words) -> bytes:
169
+ """The canonical word fold: little-endian words concatenated."""
170
+ return b"".join(struct.pack("<Q", w & MASK64) for w in words)
171
+
172
+
173
+ MATCH_INPUTS = {
174
+ "match-image-self": (PNG48, PNG48, "image"),
175
+ "match-image-distinct": (PNG48, GRAY40, "image"),
176
+ "match-image-png-vs-jpeg-same-pixels": (PNG_BASE, JPG_BASE, "image"),
177
+ "match-audio-self": (WAV, WAV, "audio"),
178
+ "match-text-self": (PROSE_A, PROSE_A, "text"),
179
+ "match-text-distinct": (PROSE_A, PROSE_B, "text"),
180
+ "match-binary-near-duplicate": (BLOB, BLOB_SHIFTED, "binary"),
181
+ "match-binary-unrelated": (BLOB, BLOB_OTHER, "binary"),
182
+ "match-video-mp4-vs-mov": (MP4, MOV, "video"),
183
+ }
184
+
185
+
186
+ class TestReferenceVectors(unittest.TestCase):
187
+ def test_detect_vectors(self):
188
+ cases = [
189
+ ("detect-png", PNG48),
190
+ ("detect-jpeg", JPG_BASE),
191
+ ("detect-bmp", BMP_SYNTH),
192
+ ("detect-wav", WAV),
193
+ ("detect-flac", FLAC),
194
+ ("detect-mp3", MP3),
195
+ ("detect-mp4", MP4),
196
+ ("detect-mov", MOV),
197
+ ("detect-pdf", PDF),
198
+ ("detect-zip", b"PK\x03\x04\x14\x00\x00\x00\x00\x00"),
199
+ ("detect-gif", b"GIF89a\x01\x00\x01\x00"),
200
+ ("detect-text", b"just some words"),
201
+ ("detect-binary", b"\x00\xff\x02\xfe"),
202
+ ]
203
+ for name, data in cases:
204
+ with self.subTest(vector=name):
205
+ rec = BY_NAME[name]
206
+ d: Detection = detect(data)
207
+ self.assertEqual(d.format, rec["format"])
208
+ self.assertEqual(d.modality, rec["modality"])
209
+ self.assertEqual(d.pending, rec["pending"])
210
+ self.assertIn(FORMATS.index(d.format), range(len(FORMATS)))
211
+ self.assertIn(MODALITIES.index(d.modality), range(len(MODALITIES)))
212
+
213
+ def assert_signature_matches(self, sig: Signature, rec: dict):
214
+ self.assertEqual(sig.modality, rec["modality"])
215
+ if sig.modality == "image":
216
+ self.assertEqual(sig.phash, int(rec["phash"], 16))
217
+ elif sig.modality == "audio":
218
+ self.assertEqual(sig.peak_count, rec["peak_count"])
219
+ self.assertEqual(sig.frames, rec["frames"])
220
+ self.assertEqual(fnv1a64(sig.peaks_bytes), int(rec["peaks_fnv1a64"], 16))
221
+ self.assertEqual(sha256_hex(sig.peaks_bytes), rec["peaks_sha256"])
222
+ elif sig.modality == "text":
223
+ self.assertEqual(sig.word_count, rec["minhash_words"])
224
+ fold = le_words(sig.words)
225
+ self.assertEqual(len(fold), sig.word_count)
226
+ self.assertEqual(fnv1a64(fold), int(rec["minhash_fnv1a64"], 16))
227
+ elif sig.modality == "binary":
228
+ self.assertEqual(sig.chunk_count, rec["chunk_count"])
229
+ self.assertEqual(fnv1a64(sig.digest_bytes), int(rec["chunks_fnv1a64"], 16))
230
+ self.assertEqual(sha256_hex(sig.digest_bytes), rec["chunks_sha256"])
231
+ else: # video
232
+ self.assertEqual(sig.frame_count, rec["frame_count"])
233
+ frames_fold = le_words(sig.frame_hashes)
234
+ self.assertEqual(len(frames_fold), sig.frame_count)
235
+ self.assertEqual(fnv1a64(frames_fold), int(rec["frames_fnv1a64"], 16))
236
+ self.assertEqual(sha256_hex(frames_fold), rec["frames_sha256"])
237
+ self.assertIsNotNone(sig.minhash)
238
+ minhash_fold = le_words(sig.minhash)
239
+ self.assertEqual(len(minhash_fold), rec["minhash_words"])
240
+ self.assertEqual(fnv1a64(minhash_fold), int(rec["minhash_fnv1a64"], 16))
241
+ self.assertEqual(sig.width, rec["width"])
242
+ self.assertEqual(sig.height, rec["height"])
243
+ self.assertEqual(f"{sig.duration_bits:016x}", rec["duration_bits"])
244
+
245
+ def test_signature_vectors(self):
246
+ cases = [
247
+ ("sig-image-phash_rgb8_48x40", PNG48),
248
+ ("sig-image-phash_gray8_40x32", GRAY40),
249
+ ("sig-image-base_444-png", PNG_BASE),
250
+ ("sig-image-base_444-jpeg", JPG_BASE),
251
+ ("sig-image-bmp-patterned-48x40", BMP_SYNTH),
252
+ ("sig-audio-tone-wav", WAV),
253
+ ("sig-audio-tone-flac", FLAC),
254
+ ("sig-audio-l3_short-mp3", MP3),
255
+ ("sig-text-prose-4kib", PROSE_A),
256
+ ("sig-binary-splitmix64-48kib", BLOB),
257
+ ("sig-binary-splitmix64-48kib-prefix-insert", BLOB_SHIFTED),
258
+ ("sig-binary-splitmix64-48kib-unrelated", BLOB_OTHER),
259
+ ("sig-video-a_64x48-mp4", MP4),
260
+ ("sig-pdf-text_page", PDF),
261
+ ]
262
+ for name, data in cases:
263
+ with self.subTest(vector=name):
264
+ self.assert_signature_matches(signature(data), BY_NAME[name])
265
+
266
+ def test_describe_vectors(self):
267
+ cases = [
268
+ ("describe-image-phash_rgb8_48x40", PNG48),
269
+ ("describe-image-bmp-patterned-48x40", BMP_SYNTH),
270
+ ("describe-audio-tone-wav", WAV),
271
+ ("describe-text-prose-4kib", PROSE_A),
272
+ ("describe-binary-splitmix64-48kib", BLOB),
273
+ ("describe-video-a_64x48-mp4", MP4),
274
+ ("describe-pdf-text_page", PDF),
275
+ ]
276
+ for name, data in cases:
277
+ with self.subTest(vector=name):
278
+ rec = BY_NAME[name]
279
+ d: Description = describe(data)
280
+ self.assertEqual(d.format, rec["format"])
281
+ self.assertEqual(d.modality, rec["modality"])
282
+ self.assertEqual(d.tier1, rec["tier1"])
283
+ self.assert_signature_matches(d.signature, rec)
284
+ facts = d.facts
285
+ if d.modality == "image":
286
+ self.assertEqual(facts["width"], rec["facts"]["width"])
287
+ self.assertEqual(facts["height"], rec["facts"]["height"])
288
+ self.assertEqual(facts["keypoints"], rec["facts"]["keypoints"])
289
+ elif d.modality == "audio":
290
+ self.assertEqual(facts["sample_rate"], rec["facts"]["sample_rate"])
291
+ self.assertEqual(facts["channels"], rec["facts"]["channels"])
292
+ self.assertEqual(facts["frames"], rec["facts"]["frames"])
293
+ self.assertEqual(facts["peaks"], rec["facts"]["peaks"])
294
+ elif d.modality == "text":
295
+ self.assertEqual(facts["words"], rec["facts"]["words"])
296
+ self.assertEqual(facts["canonical_len"], rec["facts"]["canonical_len"])
297
+ elif d.modality == "binary":
298
+ self.assertEqual(facts["len"], rec["facts"]["len"])
299
+ self.assertEqual(facts["chunks"], rec["facts"]["chunks"])
300
+ else: # video
301
+ self.assertEqual(facts["width"], rec["facts"]["width"])
302
+ self.assertEqual(facts["height"], rec["facts"]["height"])
303
+ self.assertEqual(f"{facts['duration_bits']:016x}", rec["facts"]["duration_bits"])
304
+ self.assertEqual(facts["frames_sampled"], rec["facts"]["frames_sampled"])
305
+
306
+ def test_match_vectors(self):
307
+ for name in sorted(MATCH_INPUTS):
308
+ with self.subTest(vector=name):
309
+ a, b, kind = MATCH_INPUTS[name]
310
+ rec = BY_NAME[name]
311
+ r: MatchResult = match_signatures(a, b)
312
+ self.assertEqual(r.tag, kind)
313
+ self.assertEqual(r.matched, rec["matched"])
314
+ if kind == "image":
315
+ self.assertEqual(r.slots[0], rec["hamming"])
316
+ elif kind == "audio":
317
+ if "votes" in rec and rec["votes"]:
318
+ self.assertEqual(r.slots[0], rec["votes"])
319
+ self.assertEqual(r.slots[1], rec["delta_t"] & MASK64)
320
+ else:
321
+ self.assertEqual(r.slots[0], 0)
322
+ elif kind == "video":
323
+ self.assertEqual(f"{r.slots[0]:016x}", rec["score_bits"])
324
+ self.assertEqual(f"{r.slots[1]:016x}", rec["minhash_jaccard_bits"])
325
+ else:
326
+ self.assertEqual(f"{r.slots[0]:016x}", rec["jaccard_bits"])
327
+
328
+ def test_match_cross_modality_is_pinned_refusal(self):
329
+ rec = BY_NAME["match-error-cross-modality"]
330
+ with self.assertRaises(FfiError) as ctx:
331
+ match_signatures(PNG48, PROSE_A)
332
+ self.assertEqual(ctx.exception.status, -2)
333
+
334
+ def test_error_video_non_avc_is_pinned_refusal(self):
335
+ rec = BY_NAME["error-video-non-avc"]
336
+ with self.assertRaises(FfiError) as ctx:
337
+ signature(MP4V)
338
+ self.assertEqual(ctx.exception.status, -2)
339
+
340
+ def test_corrupt_image_is_rejected(self):
341
+ corrupt = b"\x89PNG\r\n\x1a\ngarbage"
342
+ with self.assertRaises(FfiError) as ctx:
343
+ signature(corrupt)
344
+ self.assertEqual(ctx.exception.status, -2)
345
+
346
+ with self.assertRaises(FfiError) as ctx:
347
+ describe(corrupt)
348
+ self.assertEqual(ctx.exception.status, -2)
349
+
350
+ with self.assertRaises(FfiError) as ctx:
351
+ content_hash(corrupt)
352
+ self.assertEqual(ctx.exception.status, -2)
353
+
354
+ def test_malformed_pdf_is_rejected(self):
355
+ with self.assertRaises(FfiError) as ctx:
356
+ signature(b"%PDF-1.4 garbage not a pdf")
357
+ self.assertEqual(ctx.exception.status, -2)
358
+
359
+ def test_content_hash_literal_pin(self):
360
+ self.assertEqual(
361
+ content_hash(PNG48).hex(),
362
+ "1f7637f88725b1f3c1b60fb5559486349298b2a5525bac5fb1df196651e0cb74",
363
+ )
364
+
365
+
366
+ if __name__ == "__main__":
367
+ unittest.main()