python-libphash 2.0.1.0__cp314-cp314t-macosx_11_0_arm64.whl

libphash/__init__.py

@@ -0,0 +1,216 @@
+"""Perceptual image hashing: Python binding for the libphash C library.
+
+>>> import libphash as ph
+>>> h = ph.hash("cat.jpg")                      # 64-bit pHash, an int
+>>> ph.hamming_distance(h, ph.hash("cat_small.jpg"))
+>>> d = ph.hash("cat.jpg", "radial")            # a Digest
+>>> d.similarity(ph.hash("cat_rotated.jpg", "radial"))
+
+:class:`ImageContext` keeps an image and its configuration for several hashes and
+reuses its buffers across images; :func:`hash_files` hashes many files on a thread pool.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Literal, overload
+
+from ._algorithms import Algorithm, AlgorithmLike, DigestKind, to_algorithm
+from ._batch import BatchResult, ProgressCallback, hash_buffers, hash_files
+from ._config import (
+    AlphaMode,
+    Config,
+    DecodeScale,
+    WHashMode,
+    apply_config,
+    new_context,
+)
+from ._context import (
+    DigestAlgorithm,
+    DigestName,
+    ImageContext,
+    Uint64Algorithm,
+    Uint64Name,
+)
+from ._digest import (
+    PH_DIGEST_MAX_BYTES,
+    RADIAL_PCC_THRESHOLD,
+    Digest,
+    hamming_distance,
+    hash_from_hex,
+    hash_to_hex,
+    similarity,
+)
+from ._errors import PH_SUCCESS, make_error
+from ._native import ffi, lib
+from ._sources import (
+    ChannelOrder,
+    PILImage,
+    PixelArray,
+    Source,
+    StrPath,
+    SupportsArrayInterface,
+    SupportsRead,
+)
+from ._version import __version__ as __version__
+from .exceptions import (
+    AllocationError,
+    BatchCancelledError,
+    CorruptDataError,
+    DecodeError,
+    DecoderUnavailableError,
+    DigestKindError,
+    EmptyImageError,
+    ImageIOError,
+    ImageTooLargeError,
+    InvalidArgumentError,
+    NoStructureError,
+    PhashError,
+    RequiresColorError,
+    UnsupportedFormatError,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+__all__ = [
+    "PH_DIGEST_MAX_BYTES",
+    "RADIAL_PCC_THRESHOLD",
+    "Algorithm",
+    "AlgorithmLike",
+    "AllocationError",
+    "AlphaMode",
+    "BatchCancelledError",
+    "BatchResult",
+    "ChannelOrder",
+    "Config",
+    "CorruptDataError",
+    "DecodeError",
+    "DecodeScale",
+    "DecoderUnavailableError",
+    "Digest",
+    "DigestAlgorithm",
+    "DigestKind",
+    "DigestKindError",
+    "DigestName",
+    "EmptyImageError",
+    "ImageContext",
+    "ImageIOError",
+    "ImageTooLargeError",
+    "InvalidArgumentError",
+    "NoStructureError",
+    "PILImage",
+    "PhashError",
+    "PixelArray",
+    "ProgressCallback",
+    "RequiresColorError",
+    "Source",
+    "StrPath",
+    "SupportsArrayInterface",
+    "SupportsRead",
+    "Uint64Algorithm",
+    "Uint64Name",
+    "UnsupportedFormatError",
+    "WHashMode",
+    "build_info",
+    "can_decode",
+    "digest_info",
+    "hamming_distance",
+    "hash",
+    "hash_buffers",
+    "hash_files",
+    "hash_from_hex",
+    "hash_to_hex",
+    "libphash_version",
+    "similarity",
+]
+
+
+@overload
+def hash(
+    source: Source,
+    algorithm: Uint64Algorithm | Uint64Name = ...,
+    *,
+    config: Config | None = ...,
+) -> int: ...
+@overload
+def hash(
+    source: Source,
+    algorithm: DigestAlgorithm | DigestName,
+    *,
+    config: Config | None = ...,
+) -> Digest: ...
+@overload
+def hash(
+    source: Source, algorithm: AlgorithmLike, *, config: Config | None = ...
+) -> int | Digest: ...
+
+
+def hash(
+    source: Source,
+    algorithm: AlgorithmLike = Algorithm.PHASH,
+    *,
+    config: Config | None = None,
+) -> int | Digest:
+    """Hash one image.
+
+    ``source`` is a path, encoded bytes, a binary file object, an array of pixels or a
+    Pillow image. Returns an ``int`` for aHash, dHash, pHash and wHash, a
+    :class:`Digest` for the others.
+    """
+    with ImageContext(config) as ctx:
+        ctx.load(source)
+        return ctx.hash(algorithm)
+
+
+def libphash_version() -> str:
+    """Return the version of the bundled libphash (``"2.0.1"``).
+
+    It is the first three numbers of ``__version__``.
+    """
+    return str(ffi.string(lib.ph_version()).decode())
+
+
+def build_info() -> str:
+    """Return how the bundled libphash was built.
+
+    The string lists the version, decoders, threads and SIMD:
+    ``"version=2.0.1 jpeg=libjpeg-turbo png=libpng ..."``.
+    """
+    return str(ffi.string(lib.ph_get_build_info()).decode())
+
+
+def can_decode(fmt: Literal["jpeg", "png", "webp"]) -> bool:
+    """Return whether this build has a native decoder for the format.
+
+    JPEG and PNG decode without one too (through stb_image); WebP does not.
+    """
+    checks: dict[str, Callable[[], int]] = {
+        "jpeg": lib.ph_can_use_jpeg,
+        "png": lib.ph_can_use_png,
+        "webp": lib.ph_can_use_webp,
+    }
+    if fmt not in checks:
+        raise ValueError(f"fmt must be 'jpeg', 'png' or 'webp', got {fmt!r}")
+    return bool(checks[fmt]())
+
+
+def digest_info(
+    algorithm: AlgorithmLike, config: Config | None = None
+) -> tuple[int, DigestKind]:
+    """Return the size in bytes and the kind of the hashes ``algorithm`` produces.
+
+    No image is needed: the answer depends only on ``config``, which makes it fit for
+    sizing a database column, say. A 64-bit hash is 8 ``BITS`` bytes.
+    """
+    algo = to_algorithm(algorithm)
+    ctx = new_context()
+    try:
+        apply_config(config if config is not None else Config(), ctx)
+        size = ffi.new("size_t *")
+        kind = ffi.new("ph_digest_kind_t *")
+        code = lib.ph_digest_info(ctx, int(algo), size, kind)
+        if code != PH_SUCCESS:
+            raise make_error(code, context=f"digest_info({algo})")
+        return int(size[0]), DigestKind(int(kind[0]))
+    finally:
+        lib.ph_free(ctx)

libphash/__main__.py

@@ -0,0 +1,7 @@
+"""Run the command line: ``python -m libphash --help``."""
+
+import sys
+
+from ._cli import main
+
+sys.exit(main(prog="python -m libphash"))

libphash/_algorithms.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+from enum import IntEnum
+from typing import Union
+
+from ._errors import PH_SUCCESS
+from ._native import ffi, lib
+from ._sources import encode_text
+from .exceptions import InvalidArgumentError
+
+
+class Algorithm(IntEnum):
+    """A hash algorithm; the values are libphash's ``ph_algorithm_t``.
+
+    Wherever an algorithm is expected, its libphash name (``"phash"``,
+    ``"color_moments"``) is accepted too; ``str(algorithm)`` is that name.
+    """
+
+    # Literal values, so that type checkers see the members (libphash never changes an
+    # existing value; the tests compare them with the C enum).
+
+    AHASH = 0
+    DHASH = 1
+    PHASH = 2
+    WHASH = 3
+    BMH = 4
+    MHASH = 5
+    RADIAL = 6
+    COLOR_HASH = 7
+    COLOR_MOMENTS = 8
+
+    def __str__(self) -> str:
+        return str(ffi.string(lib.ph_algorithm_name(self.value)).decode())
+
+
+class DigestKind(IntEnum):
+    """What a digest's bytes mean, and so which comparisons it supports.
+
+    The values are those of ``ph_digest_kind_t``.
+    """
+
+    UNSPECIFIED = 0
+    BITS = 1
+    COEFFICIENTS = 2
+    VECTOR = 3
+    HISTOGRAM = 4
+    VECTOR16 = 5
+
+
+AlgorithmLike = Union["Algorithm", str]
+
+#: The algorithms that produce a 64-bit integer; the others produce a Digest.
+UINT64_ALGORITHMS = frozenset(
+    Algorithm[name] for name in ("AHASH", "DHASH", "PHASH", "WHASH")
+)
+
+
+def to_algorithm(value: object) -> Algorithm:
+    if isinstance(value, Algorithm):
+        return value
+    if isinstance(value, str):
+        out = ffi.new("ph_algorithm_t *")
+        if lib.ph_algorithm_from_name(encode_text(value), out) != PH_SUCCESS:
+            names = ", ".join(str(a) for a in Algorithm)
+            raise InvalidArgumentError(
+                f"unknown algorithm {value!r}; expected one of {names}"
+            )
+        return Algorithm(int(out[0]))
+    if isinstance(value, int) and not isinstance(value, bool):
+        try:
+            return Algorithm(value)
+        except ValueError:
+            pass
+    raise InvalidArgumentError(f"expected an Algorithm or its name, got {value!r}")
+
+
+def uint64_flags(algorithms: list[Algorithm]) -> int:
+    """Return the ph_hash_flags_t bits of uint64 algorithms: bit ``1 << value``."""
+    flags = 0
+    for algo in algorithms:
+        if algo not in UINT64_ALGORITHMS:
+            names = ", ".join(str(a) for a in sorted(UINT64_ALGORITHMS))
+            raise InvalidArgumentError(
+                f"{algo} produces a digest; only {names} "
+                "can be computed together or in a batch"
+            )
+        flags |= 1 << int(algo)
+    return flags

libphash/_batch.py

@@ -0,0 +1,366 @@
+from __future__ import annotations
+
+import os
+import signal
+import socket
+import threading
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+from typing import (
+    TYPE_CHECKING,
+    Generic,
+    TypeVar,
+    cast,
+)
+
+from ._algorithms import Algorithm, AlgorithmLike, to_algorithm, uint64_flags
+from ._config import Config, apply_config, new_context
+from ._errors import PH_ERR_CANCELLED, PH_SUCCESS, check, make_error
+from ._native import ffi, lib
+from ._sources import READ_FILES_IN_PYTHON, StrPath, encode_path, read_file
+from .exceptions import ImageIOError, InvalidArgumentError, PhashError
+
+if TYPE_CHECKING:
+    from typing_extensions import Self
+
+    from ._native import ph_batch_buffer_item_t, ph_batch_item_t, void_ptr
+
+#: Called as ``progress(done, total)`` after each finished item; returning ``False``
+#: stops the batch (items not yet started get BatchCancelledError).
+ProgressCallback = Callable[[int, int], bool | None]
+
+_S = TypeVar("_S")
+_P = TypeVar("_P", bound=StrPath)
+_Algorithms = AlgorithmLike | Iterable[AlgorithmLike]
+
+
+@dataclass(frozen=True)
+class BatchResult(Generic[_S]):
+    """The outcome for one item of a batch.
+
+    :func:`hash_files` gives ``BatchResult[path type]``, :func:`hash_buffers` gives
+    ``BatchResult[int]`` (the buffer's index).
+    """
+
+    #: The path or buffer index the item was given as.
+    source: _S
+    #: The hashes by algorithm, or None if the item failed.
+    hashes: dict[Algorithm, int] | None
+    #: Why the item failed, or None.
+    error: PhashError | None
+
+    @property
+    def ok(self) -> bool:
+        return self.error is None
+
+
+def hash_files(
+    paths: Iterable[_P],
+    algorithms: _Algorithms = (Algorithm.PHASH,),
+    *,
+    config: Config | None = None,
+    threads: int = 0,
+    progress: ProgressCallback | None = None,
+) -> list[BatchResult[_P]]:
+    """Hash many image files on libphash's pool of threads.
+
+    Only the 64-bit algorithms (aHash, dHash, pHash, wHash) can be batched. A failed
+    file does not stop the batch: its result carries the error. ``threads=0`` uses one
+    thread per usable CPU, ``1`` runs on the calling thread. Each worker holds one
+    decoded image at a time, so peak memory grows with the thread count (bound it with
+    ``threads`` or ``Config.max_pixels``). Ctrl+C stops the batch and raises
+    KeyboardInterrupt.
+    """
+    paths = list(paths)
+    if READ_FILES_IN_PYTHON:
+        return _files_via_buffers(paths, algorithms, config, threads, progress)
+    algos, flags = _algorithms(algorithms)
+    encoded = [ffi.new("char[]", encode_path(p)) for p in paths]
+    items = ffi.new("ph_batch_item_t[]", len(paths))
+    for item, path in zip(items, encoded, strict=True):
+        item.path = path
+    with _Run(len(paths), config, threads, progress) as run:
+        code = lib.ph_hash_files_ex(items, len(paths), flags, run.options)
+        run.finish(code)
+    return [_result(p, items[i], algos, path=p) for i, p in enumerate(paths)]
+
+
+def hash_buffers(
+    buffers: Iterable[bytes | bytearray | memoryview],
+    algorithms: _Algorithms = (Algorithm.PHASH,),
+    *,
+    config: Config | None = None,
+    threads: int = 0,
+    progress: ProgressCallback | None = None,
+) -> list[BatchResult[int]]:
+    """Hash many encoded images held in memory; see :func:`hash_files`.
+
+    ``BatchResult.source`` is the buffer's index. The buffers must not change while the
+    call runs.
+    """
+    algos, flags = _algorithms(algorithms)
+    views = [ffi.from_buffer("const uint8_t[]", b) for b in buffers]
+    items = ffi.new("ph_batch_buffer_item_t[]", len(views))
+    for item, view in zip(items, views, strict=True):
+        item.buffer = view
+        item.length = len(view)
+    with _Run(len(views), config, threads, progress) as run:
+        code = lib.ph_hash_buffers_ex(items, len(views), flags, run.options)
+        run.finish(code)
+    return [_result(i, items[i], algos) for i in range(len(views))]
+
+
+def _files_via_buffers(
+    paths: list[_P],
+    algorithms: _Algorithms,
+    config: Config | None,
+    threads: int,
+    progress: ProgressCallback | None,
+) -> list[BatchResult[_P]]:
+    """hash_files() where the binding reads the files (see READ_FILES_IN_PYTHON).
+
+    Files are read a chunk at a time, so that memory holds a few encoded files per
+    worker and not the whole batch.
+    """
+    algos, _ = _algorithms(algorithms)
+    workers = threads if threads > 0 else (os.cpu_count() or 1)
+    chunk = max(64, 4 * workers)
+    total = len(paths)
+    results: list[BatchResult[_P]] = []
+    reporter = _ChunkProgress(progress, total)
+    for start in range(0, total, chunk):
+        part = paths[start : start + chunk]
+        if reporter.stopped:
+            results += [_cancelled(p) for p in part]
+            continue
+        data: list[bytes] = []
+        failed: dict[int, PhashError] = {}
+        for i, p in enumerate(part):
+            try:
+                data.append(read_file(p))
+            except ImageIOError as e:  # noqa: PERF203 -- each file fails on its own
+                failed[i] = e
+                data.append(b"\0")  # placeholder; its result is replaced below
+        part_results = hash_buffers(
+            data,
+            algos,
+            config=config,
+            threads=threads,
+            progress=reporter.at(start),
+        )
+        for i, (p, r) in enumerate(zip(part, part_results, strict=True)):
+            if i in failed:
+                results.append(BatchResult(p, None, failed[i]))
+            else:
+                results.append(BatchResult(p, r.hashes, r.error))
+    return results
+
+
+class _ChunkProgress:
+    """The caller's progress over the chunks of one batch.
+
+    A chunk reports its items as items of the whole batch. Once progress returns False,
+    the batch stops: the chunk cancels what it has not started, and no later chunk runs.
+    """
+
+    def __init__(self, progress: ProgressCallback | None, total: int) -> None:
+        self.progress = progress
+        self.total = total
+        self.stopped = False
+
+    def at(self, offset: int) -> ProgressCallback | None:
+        progress = self.progress
+        if progress is None:
+            return None
+
+        def report(done: int, _total: int) -> bool:
+            if progress(offset + done, self.total) is False:
+                self.stopped = True
+            return not self.stopped
+
+        return report
+
+
+def _cancelled(source: _S) -> BatchResult[_S]:
+    return BatchResult(source, None, make_error(PH_ERR_CANCELLED))
+
+
+def _algorithms(algorithms: _Algorithms) -> tuple[list[Algorithm], int]:
+    if isinstance(algorithms, (str, Algorithm)):
+        algos = [to_algorithm(algorithms)]
+    else:
+        algos = sorted({to_algorithm(a) for a in algorithms})
+    if not algos:
+        raise InvalidArgumentError("no algorithm given")
+    return algos, uint64_flags(algos)
+
+
+def _result(
+    source: _S,
+    item: ph_batch_item_t | ph_batch_buffer_item_t,
+    algos: list[Algorithm],
+    path: StrPath | None = None,
+) -> BatchResult[_S]:
+    status = int(item.status)
+    if status != PH_SUCCESS:
+        return BatchResult(
+            source, None, make_error(status, context=str(source), path=path)
+        )
+    # Slots follow ascending flag bits, which is ascending Algorithm value.
+    return BatchResult(
+        source, {a: int(item.hashes[i]) for i, a in enumerate(algos)}, None
+    )
+
+
+class _Run:
+    """Options, callbacks and Ctrl+C handling around one batch call."""
+
+    def __init__(
+        self,
+        total: int,
+        config: Config | None,
+        threads: int,
+        progress: ProgressCallback | None,
+    ) -> None:
+        self.total = total
+        self.config = config if config is not None else Config()
+        self.threads = _check_threads(threads)
+        self.progress = progress
+        self.lock = threading.Lock()
+        self.done = 0
+        self.stop = False
+        self.error: BaseException | None = None
+        self.wakeup: socket.socket | None = None
+        self.options = ffi.new("ph_batch_options_t *")
+        check(lib.ph_batch_options_init(self.options))
+        self.handle = ffi.new_handle(self)
+        self.options.threads = self.threads
+        self.options.should_continue = lib._libphash_continue
+        self.options.user_data = self.handle
+        if self.progress is not None:
+            self.options.on_progress = lib._libphash_progress
+
+    def __enter__(self) -> Self:
+        template = new_context()
+        try:
+            apply_config(self.config, template)
+        except BaseException:
+            lib.ph_free(template)
+            raise
+        self.options.config = template
+        _escaped.error = None
+        self._watch_sigint()
+        return self
+
+    def __exit__(self, *exc_info: object) -> None:
+        self._unwatch_sigint()
+        lib.ph_free(self.options.config)
+        self.options.config = ffi.NULL
+
+    def finish(self, code: int) -> None:
+        """Check the batch's return code, re-raising a callback's exception."""
+        escaped, _escaped.error = _escaped.error, None
+        if escaped is not None:
+            raise escaped
+        if self.error is not None:
+            raise self.error
+        if code not in (PH_SUCCESS, PH_ERR_CANCELLED):
+            raise make_error(code, context="batch")
+
+    # Ctrl+C: while the batch runs, the main thread is inside C and Python's SIGINT
+    # handler cannot run. Its C-level handler still writes the signal number to the
+    # wakeup fd, which should_continue polls; KeyboardInterrupt follows as usual once
+    # the call returns.
+    def _watch_sigint(self) -> None:
+        if threading.current_thread() is not threading.main_thread():
+            return
+        reader, writer = socket.socketpair()
+        reader.setblocking(False)
+        writer.setblocking(False)
+        try:
+            previous = signal.set_wakeup_fd(writer.fileno(), warn_on_full_buffer=False)
+        except (ValueError, OSError):
+            reader.close()
+            writer.close()
+            return
+        if previous != -1:
+            # Someone (an event loop) owns the wakeup fd; leave it to them.
+            signal.set_wakeup_fd(previous)
+            reader.close()
+            writer.close()
+            return
+        self.wakeup = reader
+        self._writer = writer
+
+    def _unwatch_sigint(self) -> None:
+        if self.wakeup is not None:
+            signal.set_wakeup_fd(-1)
+            self.wakeup.close()
+            self._writer.close()
+            self.wakeup = None
+
+    def should_continue(self) -> bool:
+        with self.lock:
+            if self.stop:
+                return False
+            if self.wakeup is not None:
+                try:
+                    received = self.wakeup.recv(64)
+                except (BlockingIOError, InterruptedError):
+                    received = b""
+                if int(signal.SIGINT) in received:
+                    self.stop = True
+                    return False
+            return True
+
+    def on_progress(self) -> None:
+        # The callback is registered only with a progress function.
+        progress = cast("ProgressCallback", self.progress)
+        with self.lock:
+            self.done += 1
+            done = self.done
+            if self.stop:
+                return
+            try:
+                if progress(done, self.total) is False:
+                    self.stop = True
+            except BaseException as e:  # noqa: BLE001 -- re-raised from finish()
+                self.error = e
+                self.stop = True
+
+
+def _check_threads(threads: object) -> int:
+    if isinstance(threads, bool) or not isinstance(threads, int) or threads < 0:
+        raise InvalidArgumentError(f"threads must be an int >= 0, got {threads!r}")
+    return threads
+
+
+class _Escaped(threading.local):
+    error: BaseException | None = None
+
+
+# An exception that leaves a callback never reaches C: cffi passes it to
+# _keep_escaped and returns 0 to C, which stops the batch. Such an exception can
+# arrive as the callback is entered, before any line of it runs: KeyboardInterrupt,
+# when the calling thread runs the callbacks (threads=1). It is kept per thread and
+# re-raised by finish() on the calling thread; on a worker thread it only stops the
+# batch, since the calling thread cannot see it.
+_escaped = _Escaped()
+
+
+def _keep_escaped(
+    _type: type[BaseException] | None, value: BaseException | None, _tb: object
+) -> None:
+    _escaped.error = value
+
+
+@ffi.def_extern(name="_libphash_should_continue", onerror=_keep_escaped)
+def should_continue_callback(user_data: void_ptr) -> int:
+    run = cast("_Run", ffi.from_handle(user_data))
+    return 1 if run.should_continue() else 0
+
+
+@ffi.def_extern(name="_libphash_on_progress", onerror=_keep_escaped)
+def on_progress_callback(_done: int, _total: int, user_data: void_ptr) -> None:
+    run = cast("_Run", ffi.from_handle(user_data))
+    run.on_progress()