Perception 0.8.3__tar.gz → 0.8.4__tar.gz

{perception-0.8.3 → perception-0.8.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: Perception
-Version: 0.8.3
+Version: 0.8.4
 Summary: Perception provides flexible, well-documented, and comprehensively tested tooling for perceptual hashing research, development, and production use.
 License-Expression: Apache-2.0
 License-File: LICENSE
@@ -14,17 +14,17 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Provides-Extra: benchmarking
-Provides-Extra: experimental
 Provides-Extra: matching
 Provides-Extra: pdq
 Requires-Dist: Cython (>=3.0.0,<4.0.0)
 Requires-Dist: Pillow
 Requires-Dist: aiohttp ; extra == "matching"
 Requires-Dist: albumentations (>=2.0.8,<3.0.0) ; extra == "benchmarking"
-Requires-Dist: faiss-cpu (>=1.8.0,<2.0.0) ; extra == "experimental"
+Requires-Dist: faiss-cpu (>=1.8.0,<2.0.0)
 Requires-Dist: ffmpeg-python ; extra == "benchmarking"
 Requires-Dist: matplotlib ; extra == "benchmarking"
-Requires-Dist: networkit (>=11.1,<12.0.0) ; extra == "experimental"
+Requires-Dist: networkit (>=11.1,<12.0.0) ; sys_platform != "darwin"
+Requires-Dist: networkx (>=3.0,<4.0) ; sys_platform == "darwin"
 Requires-Dist: numpy (>=1.26.4,<3.0.0)
 Requires-Dist: opencv-contrib-python-headless (>=4.10.0,<5.0.0)
 Requires-Dist: pandas

{perception-0.8.3 → perception-0.8.4}/build.py

@@ -1,7 +1,6 @@
 from Cython.Build import cythonize
 import numpy as np
 
-
 compiler_directives = {"language_level": 3, "embedsignature": True}
 
 

{perception-0.8.3 → perception-0.8.4}/perception/approximate_deduplication/__init__.py

@@ -4,11 +4,12 @@ import os.path as op
 import typing
 
 import faiss
-import networkit as nk
 import numpy as np
 import tqdm
 import typing_extensions
 
+from ._graph_backend import get_graph_backend
+
 LOGGER = logging.getLogger(__name__)
 DEFAULT_PCT_PROBE = 0
 
@@ -227,16 +228,13 @@ def pairs_to_clusters(
     node_to_id_map = {v: k for k, v in id_to_node_map.items()}
 
     LOGGER.debug("Building graph.")
-    graph = nk.Graph(len(list_ids))
     node_pairs = {(id_to_node_map[pair[0]], id_to_node_map[pair[1]]) for pair in pairs}
-    for node_pair in node_pairs:
-        graph.addEdge(node_pair[0], node_pair[1])
+    backend = get_graph_backend()
+    graph = backend.build_graph(len(list_ids), node_pairs)
 
     assignments: list[ClusterAssignment] = []
     cluster_index = 0
-    cc_query = nk.components.ConnectedComponents(graph)
-    cc_query.run()
-    components = cc_query.getComponents()
+    components = backend.connected_components(graph)
 
     for component in components:
         LOGGER.debug("Got component with size: %s", len(component))
@@ -246,19 +244,9 @@ def pairs_to_clusters(
             )
             cluster_index += 1
             continue
-        # Map between node values for a connected component
-        component_node_map = dict(enumerate(component))
-        cc_sub_graph = nk.graphtools.subgraphFromNodes(graph, component, compact=True)
-        algo = nk.community.PLP(cc_sub_graph)
-        algo.run()
-        communities = algo.getPartition()
-        community_map = communities.subsetSizeMap()
-        for community, size in community_map.items():
-            LOGGER.debug("Got community with size: %s", size)
-            community_members = list(
-                communities.getMembers(community)
-            )  # Need to do this to do batching.
-            community_members = [component_node_map[i] for i in community_members]
+        communities = backend.communities(graph, component)
+        for community_members in communities:
+            LOGGER.debug("Got community with size: %s", len(community_members))
             if strictness == "community":
                 assignments.extend(
                     [
@@ -269,33 +257,20 @@ def pairs_to_clusters(
                 cluster_index += 1
                 continue
 
-            for start in range(0, len(community_members), max_clique_batch_size):
-                community_nodes = community_members[
-                    start : start + max_clique_batch_size
-                ]
-                LOGGER.debug("Creating subgraph with %s nodes.", len(community_nodes))
-                # Map between node values for a community
-                community_node_map = dict(enumerate(community_nodes))
-                subgraph = nk.graphtools.subgraphFromNodes(
-                    graph, community_nodes, compact=True
+            for clique_members in backend.maximal_cliques(
+                graph,
+                community_members,
+                max_clique_batch_size=max_clique_batch_size,
+            ):
+                assignments.extend(
+                    [
+                        {
+                            "id": node_to_id_map[n],
+                            "cluster": cluster_index,
+                        }
+                        for n in clique_members
+                    ]
                 )
-
-                while subgraph.numberOfNodes() > 0:
-                    LOGGER.debug("Subgraph size: %s", subgraph.numberOfNodes())
-                    clique = nk.clique.MaximalCliques(subgraph, maximumOnly=True)
-                    clique.run()
-                    clique_members = clique.getCliques()[0]
-                    assignments.extend(
-                        [
-                            {
-                                "id": node_to_id_map[community_node_map[n]],
-                                "cluster": cluster_index,
-                            }
-                            for n in clique_members
-                        ]
-                    )
-                    cluster_index += 1
-                    for n in clique_members:
-                        subgraph.removeNode(n)
+                cluster_index += 1
 
     return assignments

perception-0.8.4/perception/approximate_deduplication/_graph_backend.py

@@ -0,0 +1,138 @@
+import sys
+import typing
+from abc import ABC, abstractmethod
+
+
+class GraphBackend(ABC):
+    @abstractmethod
+    def build_graph(
+        self, node_count: int, edges: typing.Iterable[tuple[int, int]]
+    ) -> typing.Any: ...
+
+    @abstractmethod
+    def connected_components(self, graph: typing.Any) -> list[list[int]]: ...
+
+    @abstractmethod
+    def communities(
+        self, graph: typing.Any, component: list[int]
+    ) -> list[list[int]]: ...
+
+    @abstractmethod
+    def maximal_cliques(
+        self,
+        graph: typing.Any,
+        community_nodes: list[int],
+        max_clique_batch_size: int,
+    ) -> list[list[int]]: ...
+
+
+class NetworkitGraphBackend(GraphBackend):
+    def __init__(self):
+        import networkit as nk
+
+        self.nk = nk
+
+    def build_graph(
+        self, node_count: int, edges: typing.Iterable[tuple[int, int]]
+    ) -> typing.Any:
+        graph = self.nk.Graph(node_count)
+        for start, end in edges:
+            graph.addEdge(start, end)
+        return graph
+
+    def connected_components(self, graph: typing.Any) -> list[list[int]]:
+        cc_query = self.nk.components.ConnectedComponents(graph)
+        cc_query.run()
+        return cc_query.getComponents()
+
+    def communities(self, graph: typing.Any, component: list[int]) -> list[list[int]]:
+        component_node_map = dict(enumerate(component))
+        subgraph = self.nk.graphtools.subgraphFromNodes(graph, component, compact=True)
+        algo = self.nk.community.PLP(subgraph, maxIterations=32)
+        algo.run()
+        communities = algo.getPartition()
+        return [
+            [component_node_map[node] for node in communities.getMembers(community)]
+            for community in communities.subsetSizeMap().keys()
+        ]
+
+    def maximal_cliques(
+        self,
+        graph: typing.Any,
+        community_nodes: list[int],
+        max_clique_batch_size: int,
+    ) -> list[list[int]]:
+        cliques: list[list[int]] = []
+        for start in range(0, len(community_nodes), max_clique_batch_size):
+            batch_nodes = community_nodes[start : start + max_clique_batch_size]
+            community_node_map = dict(enumerate(batch_nodes))
+            subgraph = self.nk.graphtools.subgraphFromNodes(
+                graph, batch_nodes, compact=True
+            )
+
+            while subgraph.numberOfNodes() > 0:
+                clique = self.nk.clique.MaximalCliques(subgraph, maximumOnly=True)
+                clique.run()
+                clique_members = clique.getCliques()[0]
+                cliques.append([community_node_map[node] for node in clique_members])
+                for node in clique_members:
+                    subgraph.removeNode(node)
+
+        return cliques
+
+
+class NetworkxGraphBackend(GraphBackend):
+    def __init__(self):
+        import networkx as nx
+
+        self.nx = nx
+
+    def build_graph(
+        self, node_count: int, edges: typing.Iterable[tuple[int, int]]
+    ) -> typing.Any:
+        graph = self.nx.Graph()
+        graph.add_nodes_from(range(node_count))
+        graph.add_edges_from(edges)
+        return graph
+
+    def connected_components(self, graph: typing.Any) -> list[list[int]]:
+        return [list(component) for component in self.nx.connected_components(graph)]
+
+    def communities(self, graph: typing.Any, component: list[int]) -> list[list[int]]:
+        subgraph = graph.subgraph(component)
+        return [
+            list(community)
+            for community in self.nx.algorithms.community.asyn_lpa_communities(
+                subgraph, seed=0
+            )
+        ]
+
+    def maximal_cliques(
+        self,
+        graph: typing.Any,
+        community_nodes: list[int],
+        max_clique_batch_size: int,
+    ) -> list[list[int]]:
+        cliques: list[list[int]] = []
+        for start in range(0, len(community_nodes), max_clique_batch_size):
+            batch_nodes = community_nodes[start : start + max_clique_batch_size]
+            subgraph = graph.subgraph(batch_nodes).copy()
+
+            while subgraph.number_of_nodes() > 0:
+                clique_members = max(
+                    self.nx.find_cliques(subgraph),
+                    key=lambda clique: (
+                        len(clique),
+                        tuple(sorted(clique)),
+                    ),
+                )
+                cliques.append(list(clique_members))
+                subgraph.remove_nodes_from(clique_members)
+
+        return cliques
+
+
+def get_graph_backend() -> GraphBackend:
+    if sys.platform == "darwin":
+        return NetworkxGraphBackend()
+    return NetworkitGraphBackend()

{perception-0.8.3 → perception-0.8.4}/perception/approximate_deduplication/debug.py

@@ -79,10 +79,8 @@ def vizualize_pair(
             circle_size=circle_size,
         )
     else:
-        LOGGER.warning(
-            """No match_metadata provided, recalculating match points,
-            won't match perception match points."""
-        )
+        LOGGER.warning("""No match_metadata provided, recalculating match points,
+            won't match perception match points.""")
         img_matched = viz_brute_force(features_1, features_2, img1, img2, ratio=ratio)
 
     return img_matched

{perception-0.8.3 → perception-0.8.4}/perception/benchmarking/image_transforms.py

@@ -17,7 +17,7 @@ def apply_watermark(watermark, alpha: float = 1.0, size: float = 1.0):
 
     # Why do we have to do this? It's not clear. But the process doesn't work
     # without it.
-    (B, G, R, A) = cv2.split(watermark)
+    B, G, R, A = cv2.split(watermark)
     B = cv2.bitwise_and(B, B, mask=A)
     G = cv2.bitwise_and(G, G, mask=A)
     R = cv2.bitwise_and(R, R, mask=A)
@@ -25,7 +25,7 @@ def apply_watermark(watermark, alpha: float = 1.0, size: float = 1.0):
 
     def transform(image):
         # Add alpha channel
-        (h, w) = image.shape[:2]
+        h, w = image.shape[:2]
         wh, ww = watermark.shape[:2]
         scale = size * min(h / wh, w / ww)
         image = np.dstack([image, np.ones((h, w), dtype="uint8") * 255])

{perception-0.8.3 → perception-0.8.4}/perception/hashers/__init__.py

@@ -7,7 +7,6 @@ from .image.wavelet import WaveletHash
 from .video.framewise import FramewiseHasher
 from .video.tmk import TMKL1, TMKL2
 
-
 __all__ = [
     "ImageHasher",
     "VideoHasher",

{perception-0.8.3 → perception-0.8.4}/pyproject.toml

@@ -10,6 +10,9 @@ dependencies = [
   "Cython>=3.0.0,<4.0.0",
   "numpy>=1.26.4,<3.0.0",
   "opencv-contrib-python-headless>=4.10.0,<5.0.0",
+  "faiss-cpu>=1.8.0,<2.0.0",
+  "networkit>=11.1,<12.0.0; sys_platform != 'darwin'",
+  "networkx>=3.0,<4.0; sys_platform == 'darwin'",
   "pandas",
   "Pillow",
   "pywavelets>=1.5.0,<2.0.0",
@@ -18,7 +21,7 @@ dependencies = [
   "scipy",
   "tqdm>=4.67.1,<5.0.0",
 ]
-version = "0.8.3"
+version = "0.8.4"
 
 
 [project.optional-dependencies]
@@ -30,7 +33,6 @@ benchmarking = [
   "ffmpeg-python",
 ]
 matching = ["aiohttp", "python-json-logger"]
-experimental = ["networkit>=11.1,<12.0.0", "faiss-cpu>=1.8.0,<2.0.0"]
 pdq = ["pdqhash>=0.2.7,<0.3.0"]
 
 
@@ -38,7 +40,7 @@ pdq = ["pdqhash>=0.2.7,<0.3.0"]
 
 
 [tool.poetry.group.dev.dependencies]
-black = "^24"
+black = "^26"
 coverage = "*"
 ipython = "*"
 mypy = "*"

{perception-0.8.3 → perception-0.8.4}/setup.py

@@ -16,6 +16,7 @@ package_data = \
 install_requires = \
 ['Cython>=3.0.0,<4.0.0',
  'Pillow',
+ 'faiss-cpu>=1.8.0,<2.0.0',
  'numpy>=1.26.4,<3.0.0',
  'opencv-contrib-python-headless>=4.10.0,<5.0.0',
  'pandas',
@@ -26,18 +27,19 @@ install_requires = \
  'validators>=0.22.0,<1.0.0']
 
 extras_require = \
-{'benchmarking': ['matplotlib',
+{':sys_platform != "darwin"': ['networkit>=11.1,<12.0.0'],
+ ':sys_platform == "darwin"': ['networkx>=3.0,<4.0'],
+ 'benchmarking': ['matplotlib',
                   'albumentations>=2.0.8,<3.0.0',
                   'tabulate',
                   'scikit-learn',
                   'ffmpeg-python'],
- 'experimental': ['networkit>=11.1,<12.0.0', 'faiss-cpu>=1.8.0,<2.0.0'],
  'matching': ['aiohttp', 'python-json-logger'],
  'pdq': ['pdqhash>=0.2.7,<0.3.0']}
 
 setup_kwargs = {
     'name': 'Perception',
-    'version': '0.8.3',
+    'version': '0.8.4',
     'description': 'Perception provides flexible, well-documented, and comprehensively tested tooling for perceptual hashing research, development, and production use.',
     'long_description': "# perception ![ci](https://github.com/thorn-oss/perception/workflows/ci/badge.svg)\n\n`perception` provides flexible, well-documented, and comprehensively tested tooling for perceptual hashing research, development, and production use. See [the documentation](https://perception.thorn.engineering/en/latest/) for details.\n\n## Background\n\n`perception` was initially developed at [Thorn](https://www.thorn.org) as part of our work to eliminate child sexual abuse material from the internet. For more information on the issue, check out [our CEO's TED talk](https://www.thorn.org/blog/time-is-now-eliminate-csam/).\n\n## Getting Started\n\n### Installation\n\n`pip install perception`\n\n### Hashing\n\nHashing with different functions is simple with `perception`.\n\n```python\nfrom perception import hashers\n\nfile1, file2 = 'test1.jpg', 'test2.jpg'\nhasher = hashers.PHash()\nhash1, hash2 = hasher.compute(file1), hasher.compute(file2)\ndistance = hasher.compute_distance(hash1, hash2)\n```\n\n### Examples\n\nSee below for end-to-end examples for common use cases for perceptual hashes.\n\n- [Detecting child sexual abuse material](https://perception.thorn.engineering/en/latest/examples/detecting_csam.html)\n- [Deduplicating media](https://perception.thorn.engineering/en/latest/examples/deduplication.html)\n- [Benchmarking perceptual hashes](https://perception.thorn.engineering/en/latest/examples/benchmarking.html)\n\n## Supported Hashing Algorithms\n\n`perception` currently ships with:\n\n- pHash (DCT hash) (`perception.hashers.PHash`)\n- Facebook's PDQ Hash (`perception.hashers.PDQ`)\n- dHash (difference hash) (`perception.hashers.DHash`)\n- aHash (average hash) (`perception.hashers.AverageHash`)\n- Marr-Hildreth (`perception.hashers.MarrHildreth`)\n- Color Moment (`perception.hashers.ColorMoment`)\n- Block Mean (`perception.hashers.BlockMean`)\n- wHash (wavelet hash) (`perception.hashers.WaveletHash`)\n\n## Contributing\n\nTo work on the project, start by doing the following.\n\n```bash\n# Install local dependencies for\n# code completion, etc.\nmake init\n\n- To do a (close to) comprehensive check before committing code, you can use `make precommit`.\n\nTo implement new features, please first file an issue proposing your change for discussion.\n\nTo report problems, please file an issue with sample code, expected results, actual results, and a complete traceback.\n\n## Alternatives\n\nThere are other packages worth checking out to see if they meet your needs for perceptual hashing. Here are some\nexamples.\n\n- [dedupe](https://github.com/dedupeio/dedupe)\n- [imagededup](https://idealo.github.io/imagededup/)\n- [ImageHash](https://github.com/JohannesBuchner/imagehash)\n- [PhotoHash](https://github.com/bunchesofdonald/photohash)\n```\n",
     'author': 'Thorn',