panopticml 0.0.2__py3-none-any.whl

panopticml/compute/__init__.py

@@ -0,0 +1 @@
+from .clustering import make_clusters

panopticml/compute/clustering.py

@@ -0,0 +1,122 @@
+import faiss
+import numpy as np
+import torch
+
+from panoptic.models import Vector
+from panoptic.models.results import ScoreList, Score, Group
+from ..utils import similarity_matrix
+
+
+def make_clusters(vectors: list[Vector], **kwargs) -> (list[list[str]], list[int]):
+    res_clusters = []
+    res_distances = []
+    vectors, sha1 = zip(*[(i.data, i.sha1) for i in vectors])
+    sha1 = np.asarray(sha1)
+    clusters: np.ndarray
+
+    clusters, distances = _make_clusters_faiss(vectors, **kwargs)
+
+    for cluster in list(set(clusters)):
+        sha1_cluster = sha1[clusters == cluster]
+        current_cluster_distances = distances[clusters == cluster]
+        if distances is not None:
+            res_distances.append(np.mean(current_cluster_distances))
+        res_clusters.append(list(sha1_cluster))
+    # sort clusters by distances
+    sorted_clusters = [cluster for _, cluster in sorted(zip(res_distances, res_clusters))]
+    return sorted_clusters, sorted(res_distances)
+
+
+def _make_clusters_faiss(vectors, nb_clusters=6, **kwargs) -> (np.ndarray, np.ndarray):
+    def _make_single_kmean(vectors, nb_clusters):
+        kmean = faiss.Kmeans(vectors.shape[1], nb_clusters, niter=20, verbose=False)
+        kmean.train(vectors)
+        return kmean.index.search(vectors, 1)
+
+    vectors = np.asarray(vectors)
+    if nb_clusters == -1:
+        import hdbscan
+        clusterer = hdbscan.HDBSCAN(min_cluster_size=5, gen_min_span_tree=True)
+        clusterer.fit(vectors)
+        indices = clusterer.labels_
+        probabilities = clusterer.probabilities_
+        distances = np.zeros_like(probabilities, dtype=np.float32)
+        unique_clusters = np.unique(indices)
+        # compute distances just like the one returned by kmeans to have consistent metrics
+        for cluster_id in unique_clusters:
+            if cluster_id == -1:
+                distances[indices == -1] = 100.0
+                continue
+            cluster_mask = (indices == cluster_id)
+            cluster_vectors = vectors[cluster_mask]
+            cluster_probabilities = probabilities[cluster_mask]
+            center_local_index = np.argmax(cluster_probabilities)
+            center_vector = cluster_vectors[center_local_index].reshape(1, -1)
+            dists = faiss.pairwise_distances(center_vector, cluster_vectors)[0]
+            distances[cluster_mask] = dists
+    else:
+        distances, indices = _make_single_kmean(vectors, nb_clusters)
+    return indices.flatten(), distances.flatten()
+
+
+def cluster_by_text(image_vectors: list[Vector], text_vectors: list[np.array], text_labels: list[str]) -> list[Group]:
+    vectors, sha1s = zip(*[(i.data, i.sha1) for i in image_vectors])
+    sha1s_array = np.asarray(sha1s)
+
+    similarities, closest_text_indices = similarity_matrix(vectors, text_vectors)
+
+    clusters = []
+    distances = []
+    clusters_text = []
+    cluster_sims = []
+
+    for text_index in closest_text_indices.unique():
+        clusters_text.append(text_labels[text_index])
+        cluster = sha1s_array[closest_text_indices == text_index]
+
+        # similarities of each image inside the cluster and the text
+        cluster_sim = similarities[closest_text_indices == text_index]
+        sorted_sim = cluster_sim.sort(descending=True).values
+        cluster_sims.append([round(x, 2) for x in sorted_sim.tolist()])
+
+        # sort cluster by descending similarity
+        sorting_index = cluster_sim.argsort(descending=True)
+        sorted_cluster = cluster[sorting_index]
+        if type(sorted_cluster) is not np.ndarray:
+            sorted_cluster = np.array([sorted_cluster])
+        clusters.append(sorted_cluster)
+
+        # compute mean distance of the images
+        distance = float((1 - torch.mean(cluster_sim)) * 100)
+        distances.append(distance)
+
+    groups = []
+    for cluster, distance, name, cluster_sim in zip(clusters, distances, clusters_text, cluster_sims):
+        similarities = ScoreList(min=0, max=1, max_is_best=True, values=cluster_sim)
+        group = Group(score=Score(distance, min=0, max=200, max_is_best=False,
+                                  description="Mean distance between the images of this cluster and the queried text"),
+                      scores=similarities)
+        group.sha1s = list(cluster)
+        group.name = f"Cluster {name}"
+        groups.append(group)
+
+    return groups
+
+def custom_range(min_i, max_i, steps, increments):
+    """
+    Generate a range of values from min_i to max_i with a variable increment for each step
+    :param min_i: first value
+    :param max_i: last value
+    :param steps: values for which increment should change
+    :param increments: increments values
+    :return:
+    """
+    i = min_i
+    current_step = 0
+    current_incr = 1
+    while i < max_i:
+        yield i
+        if i >= steps[current_step] and current_step < len(steps) - 1:
+            current_incr = increments[current_step]
+            current_step += 1
+        i += current_incr

panopticml/compute/faiss_tree.py

@@ -0,0 +1,72 @@
+import os
+import pickle
+
+import faiss
+import numpy as np
+
+from .similarity import get_text_vectors
+from ..models import VectorType
+from panoptic.core.plugin.plugin import APlugin
+
+
+class FaissTree:
+    def __init__(self, index: faiss.IndexFlatIP, labels: list[str]):
+        self.index = index
+        self.labels = labels
+
+    def query(self, vectors: list[np.ndarray], k=999999):
+        vector_center = np.mean(vectors, axis=0)
+        vector = np.asarray([vector_center])
+
+        real_k = min(k, len(self.labels))
+        vector = vector.reshape(1, -1)
+        dist, ind = self.index.search(vector, real_k)
+        indices = [x for x in ind[0]]
+        distances = [x for x in dist[0]]  # avoid some strange overflow behavior
+        return [{'sha1': self.labels[i], 'dist': float('%.2f' % (distances[index]))} for index, i in
+                enumerate(indices)]
+
+    def query_texts(self, texts: list[str], transformer):
+        text_vectors = get_text_vectors(texts, transformer)
+        return self.query(text_vectors)
+
+
+def gen_tree_file_name(vec_type: VectorType):
+    return f"{vec_type.value}_faiss_tree.pkl"
+
+
+async def create_faiss_tree(plugin: APlugin, vec_type: VectorType):
+    project = plugin.project
+    name = gen_tree_file_name(vec_type)
+    vectors = await project.get_vectors(plugin.name, vector_type=vec_type.value)
+
+    if vectors is None or len(vectors) == 0:
+        return
+    vec_data, sha1_list = zip(*[(i.data, i.sha1) for i in vectors])
+    vec_np_arr = np.asarray(vec_data)
+    faiss.normalize_L2(vec_np_arr)
+
+    # create the faiss index based on this post: https://anttihavanko.medium.com/building-image-search-with-openai-clip-5a1deaa7a6e2
+    vector_size = vec_np_arr.shape[1]
+    index = faiss.IndexFlatIP(vector_size)
+    # faiss.ParameterSpace().set_index_parameter(index, 'nprobe', 100)
+    index.add(np.asarray(vec_np_arr))
+
+    tree = FaissTree(index, sha1_list)
+
+    with open(os.path.join(plugin.data_path, name), 'wb') as f:
+        pickle.dump(tree, f)
+
+    return FaissTree(index=index, labels=sha1_list)
+
+
+def load_faiss_tree(plugin: APlugin, vec_type: VectorType) -> FaissTree | None:
+    name = gen_tree_file_name(vec_type)
+    path = os.path.join(plugin.data_path, name)
+    if not os.path.exists(path):
+        return None
+    try:
+        with open(path, 'rb') as f:
+            return pickle.load(f)
+    except ModuleNotFoundError:
+        return None

panopticml/compute/similarity.py

@@ -0,0 +1,11 @@
+import numpy as np
+
+
+def get_text_vectors(texts: [str], transformer):
+    vectors = []
+    if transformer.can_handle_text:
+        for text in texts:
+            vectors.append(transformer.to_text_vector(text))
+    else:
+        raise ValueError(f"The selected transformer {transformer.name_or_path} does not support text vectors.")
+    return np.asarray(vectors)

panopticml/compute/transformers/__init__.py

@@ -0,0 +1,131 @@
+from enum import Enum
+
+import torch
+from PIL import Image
+import numpy as np
+
+from ....panopticml.utils import resolve_device
+
+
+class TransformerName(Enum):
+    mobilenet = "mobilenet"
+    clip = "clip"
+    siglip = "siglip"
+    dinov2 = "dinov2"
+    auto = "auto"
+
+def get_transformer(model: TransformerName=TransformerName.clip, hugging_face_model=None):
+    match model:
+        case TransformerName.mobilenet:
+            return GoogleTransformer()
+        case TransformerName.clip:
+            return CLIPTransformer()
+        case TransformerName.siglip:
+            return SIGLIPTransformer()
+        case TransformerName.dinov2:
+            return Dinov2Transformer()
+        case TransformerName.auto:
+            return AutoTransformer(hugging_face_model)
+
+class Transformer(object):
+    def __init__(self):
+        import torch
+        from transformers import logging
+        logging.set_verbosity_error()
+        self.device = resolve_device()
+        self.tokenizer = None
+        self.processor = None
+        self.model = None
+
+    @property
+    def can_handle_text(self):
+        return False
+
+class AutoTransformer(Transformer):
+    def __init__(self, hugging_face_model=None):
+        super().__init__()
+        from transformers import AutoModel, AutoProcessor
+        if hugging_face_model:
+            self.model = AutoModel.from_pretrained(hugging_face_model).to(self.device)
+            self.processor = AutoProcessor.from_pretrained(hugging_face_model)
+            self.name = hugging_face_model
+
+    @property
+    def can_handle_text(self):
+        return True
+
+    def to_vector(self, image: Image) -> np.ndarray:
+        inputs = self.processor(images=[image], return_tensors="pt")
+        inputs = {k: v.to(self.device) for k, v in inputs.items()}
+        with torch.no_grad():
+            image_features = self.model.get_image_features(**inputs)
+            image_features = image_features / image_features.norm(dim=-1, keepdim=True)
+        return image_features.cpu().numpy().flatten()
+
+    def to_text_vector(self, text: str) -> np.ndarray:
+        inputs = self.processor(text=[text], return_tensors="pt")
+        inputs = {k: v.to(self.device) for k, v in inputs.items()}
+        with torch.no_grad():
+            text_features = self.model.get_text_features(**inputs)
+            text_features = text_features / text_features.norm(dim=-1, keepdim=True)
+        return text_features.cpu().numpy().flatten()
+
+class GoogleTransformer(Transformer):
+    def __init__(self):
+        super().__init__()
+        from transformers import MobileNetV2Model, AutoImageProcessor
+        self.model = MobileNetV2Model.from_pretrained("google/mobilenet_v2_1.0_224")
+        self.processor = AutoImageProcessor.from_pretrained("google/mobilenet_v2_1.0_224")
+        self.name = "MobileNetV2"
+
+    @property
+    def can_handle_text(self):
+        return False
+
+    def to_vector(self, image: Image) -> np.ndarray:
+        input1 = self.processor(images=image, return_tensors="pt")
+        output1 = self.model(**input1)
+        pooled_output1 = output1[1].detach().numpy()
+        vector = pooled_output1.flatten()
+        return vector
+
+class CLIPTransformer(AutoTransformer):
+    def __init__(self):
+        model_name = "openai/clip-vit-base-patch32"
+        super().__init__(model_name)
+        self.name = "CLIP"
+
+
+    @property
+    def can_handle_text(self):
+        return True
+
+class SIGLIPTransformer(AutoTransformer):
+    def __init__(self):
+        model_name = "google/siglip2-so400m-patch16-naflex"
+        super().__init__(model_name)
+        self.name = "SIGLIP"
+
+    @property
+    def can_handle_text(self):
+        return True
+
+
+class Dinov2Transformer(Transformer):
+    def __init__(self):
+        super().__init__()
+        from transformers import AutoModel, AutoImageProcessor
+        ckpt = "facebook/dinov2-base"
+        self.model = AutoModel.from_pretrained(ckpt).to(self.device)
+        self.processor = AutoImageProcessor.from_pretrained(ckpt, use_fast=True)
+        self.name = "Dinov2"
+
+    @property
+    def can_handle_text(self):
+        return False
+
+    def to_vector(self, image: Image) -> np.ndarray:
+        inputs = self.processor(images=image, return_tensors="pt").to(self.device)
+        with torch.no_grad():
+            outputs = self.model(**inputs)
+        return outputs.last_hidden_state.mean(dim=1).cpu().numpy()[0]

panopticml/compute_vector_task.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+from .utils import preprocess_image
+from .models import VectorType
+
+if TYPE_CHECKING:
+    from . import PanopticML
+
+import aiofiles
+
+from panoptic.core.task.task import Task
+from panoptic.models import Instance, Vector
+
+logger = logging.getLogger('PanopticML:VectorTask')
+
+
+class ComputeVectorTask(Task):
+    def __init__(self, plugin: PanopticML, source: str, type_: VectorType, instance: Instance,
+                 data_path: str):
+        super().__init__()
+        self.plugin = plugin
+        self.project = plugin.project
+        self.source = source
+        self.type = type_
+        self.instance = instance
+        self.transformer = self.plugin.transformer
+        self.name = f'{self.transformer.name} Vectors ({type_.value})'
+        self.data_path = data_path
+        self.key += f"_{type_.value}"
+
+    async def run(self):
+        instance = self.instance
+        exist = await self.project.vector_exist(self.source, self.type.value, instance.sha1)
+        if exist:
+            return
+
+        image_data = await self.project.get_project().db.get_large_image(instance.sha1)
+        if not image_data:
+            file = instance.url
+            async with aiofiles.open(file, mode='rb') as file:
+                image_data = await file.read()
+
+        vector_data = None
+        vector_data = await self._async(self.compute_image_clip, image_data)
+
+        if vector_data is None:
+            return
+        vector = Vector(self.source, self.type.name, instance.sha1, vector_data)
+        res = await self.project.add_vector(vector)
+        del vector
+        return res
+
+    async def run_if_last(self):
+        await self.plugin._update_tree(self.type)
+
+    def compute_image_clip(self, image_data: bytes):
+        image = preprocess_image(image_data, self.type)
+        vector = self.transformer.to_vector(image)
+
+        del image
+        return vector

panopticml/models.py

@@ -0,0 +1,6 @@
+from enum import Enum
+
+
+class VectorType(Enum):
+    rgb = 'rgb'
+    greyscale = 'greyscale'

panopticml/panoptic_ml.py

@@ -0,0 +1,340 @@
+from typing import Dict
+
+import numpy as np
+import requests
+from PIL import Image
+from pydantic import BaseModel
+
+from panoptic.core.plugin.plugin import APlugin
+from panoptic.core.plugin.plugin_project_interface import PluginProjectInterface
+from panoptic.models import Instance, ActionContext, PropertyId, PropertyType
+from panoptic.models.results import Group, ActionResult, Notif, NotifType, NotifFunction, ScoreList, Score
+from panoptic.utils import group_by_sha1
+
+from .compute import make_clusters
+from .compute.clustering import cluster_by_text
+from .compute.faiss_tree import load_faiss_tree, create_faiss_tree, FaissTree
+from .compute.similarity import get_text_vectors
+from .compute.transformers import get_transformer, TransformerName
+from .compute_vector_task import ComputeVectorTask
+from .models import VectorType
+from .utils import is_image_url
+
+
+class PluginParams(BaseModel):
+    """
+    @greyscale: if this is checked, vectors can be recomputed but this time images will be converted to greyscale before
+    @model: the name of the transformer to user, values: clip | mobilenet | siglip | auto, default to clip, if using auto, need to specify a huggingface model name, for instance: google/siglip2-base-patch16-224
+    """
+    greyscale: bool = False
+    model: str = "clip"
+    hugging_face_model: str = None
+
+
+class PanopticML(APlugin):
+    """
+    Default Machine Learning plugin for Panoptic
+    Uses CLIP to generate vectors and FAISS for clustering / similarity functions
+    """
+
+    def __init__(self, project: PluginProjectInterface, plugin_path: str, name: str):
+        super().__init__(name=name, project=project, plugin_path=plugin_path)
+        self.params: PluginParams = PluginParams()
+
+        self.project.on_instance_import(self.compute_image_vectors_on_import)
+        self.project.on_folder_delete(self.rebuild_trees)
+        self.add_action_easy(self.find_images, ['similar'])
+        self.add_action_easy(self.compute_clusters, ['group'])
+        self.add_action_easy(self.cluster_by_tags, ['group'])
+        self.add_action_easy(self.find_duplicates, ['group'])
+        self.add_action_easy(self.search_by_text, ['execute'])
+        self._comp_vec_desc = self.add_action_easy(self.compute_vectors, ['execute'])
+        self._comp_all_vec_desc = self.add_action_easy(self.compute_all_vectors, ['execute'])
+
+        self.trees: Dict[VectorType, FaissTree] = {}
+        self._transformer = None
+
+    @property
+    def transformer(self):
+        if self._transformer is None:
+            self._transformer = self._load_transformer()
+        return self._transformer
+
+    async def start(self):
+        await super().start()
+
+        [await self._get_tree(t) for t in VectorType]
+
+    def _get_vector_func_notifs(self, vec_type: VectorType):
+        res = [
+            NotifFunction(self._comp_vec_desc.id,
+                          ActionContext(ui_inputs={"vec_type": vec_type.value}),
+                          message=f"Compute all vectors of type {vec_type.value}"),
+            NotifFunction(self._comp_all_vec_desc.id,
+                          ActionContext(),
+                          message="Compute vectors off all types")
+        ]
+        return res
+
+    async def compute_vectors(self, context: ActionContext, vec_type: VectorType):
+        """
+        Compute image vectors of selected vector type
+        """
+
+        instances = await self.project.get_instances(ids=context.instance_ids)
+        for i in instances:
+            await self._compute_image_vector(i, vec_type)
+
+        notif = Notif(type=NotifType.INFO,
+                      name="ComputeVector",
+                      message=f"Successfuly started compute of vectors of type {vec_type.value}")
+        return ActionResult(notifs=[notif])
+
+    async def compute_image_vectors_on_import(self, instance: Instance):
+        await self._compute_image_vector(instance, VectorType.rgb)
+        if self.params.greyscale:
+            await self._compute_image_vector(instance, VectorType.greyscale)
+
+    async def compute_all_vectors(self, context: ActionContext):
+        """
+        Compute image vectors of all supported types
+        """
+        res = [await self.compute_vectors(context, t) for t in VectorType]
+        return ActionResult(notifs=[n for r in res for n in r.notifs])
+
+    async def _compute_image_vector(self, instance: Instance, vector: VectorType):
+        task = ComputeVectorTask(self, self.name, vector, instance, self.data_path)
+        self.project.add_task(task)
+
+    async def compute_clusters(self, context: ActionContext, vec_type: VectorType = VectorType.rgb,
+                               nb_clusters: int = 10, label_clusters: bool = False):
+        """
+        Computes images clusters with Faiss Kmeans
+        @nb_clusters: requested number of clusters
+        """
+        instances = await self.project.get_instances(context.instance_ids)
+        sha1_to_instance = group_by_sha1(instances)
+        sha1_to_ahash = {i.sha1: i.ahash for i in instances}
+        sha1s = list(sha1_to_instance.keys())
+
+        if not sha1s:
+            empty_notif = Notif(NotifType.ERROR, name="NoData", message="No instance found")
+            return ActionResult(notifs=[empty_notif])
+
+        vectors = await self.project.get_vectors(source=self.name, vector_type=vec_type.value, sha1s=sha1s)
+
+        if not vectors:
+            empty_notif = Notif(NotifType.ERROR,
+                                name="NoData",
+                                message=f"""For the clustering function image vectors are needed.
+                                        No such vectors ({vec_type.value}) could be found. 
+                                        Compute the vectors and try again.) """,
+                                functions=self._get_vector_func_notifs(vec_type))
+            return ActionResult(notifs=[empty_notif])
+        clusters, distances = make_clusters(vectors, method="kmeans", nb_clusters=nb_clusters)
+        groups = []
+        groups_images = []
+        labels = []
+        i = 0
+        # TODO: put back mistral when it's working properly
+        if label_clusters:
+            from .mistral_test import create_labels_from_group, generate_group_image
+        for cluster, distance in zip(clusters, distances):
+            group = Group(score=Score(min=0, max=100, max_is_best=False, value=distance))
+            if label_clusters:
+                images = [sha1_to_instance[sha1][0].url for sha1 in cluster[:20]]
+                groups_images.append(generate_group_image(images, i))
+                i += 1
+            group.sha1s = sorted(cluster, key=lambda sha1: sha1_to_ahash[sha1])
+            groups.append(group)
+        if len(groups_images) > 0:
+            labels = create_labels_from_group(groups_images)
+        for i, g in enumerate(groups):
+            g.name = f"Cluster {i}" if not len(labels) > 0 else "-".join(labels[i])
+
+        return ActionResult(groups=groups)
+
+    async def find_images(self, context: ActionContext, vec_type: VectorType = VectorType.rgb):
+        """
+        Find Similar images using Cosine distances.
+        dist: 0 -> images are considered highly dissimilar
+        dist: 1 -> images are considered identical
+        See: https://en.wikipedia.org/wiki/Cosine_similarity for more.
+        """
+        instances = await self.project.get_instances(context.instance_ids)
+        sha1s = [i.sha1 for i in instances]
+        ignore_sha1s = set(sha1s)
+        vectors = await self.project.get_vectors(source=self.name, vector_type=vec_type.value, sha1s=sha1s)
+
+        if not vectors:
+            return ActionResult(notifs=[Notif(
+                NotifType.ERROR,
+                name="NoData",
+                message=f"""For the similarity function image vectors are needed.
+                            No such vectors ({vec_type.value}) could be found. 
+                            Compute the vectors and try again.) """,
+                functions=self._get_vector_func_notifs(vec_type))])
+
+        vector_datas = [x.data for x in vectors]
+
+        tree = await self._get_tree(vec_type)
+        if not tree:
+            notif = Notif(type=NotifType.ERROR, name="NoFaissTree",
+                          message=f"No Faiss tree could be loaded for vec_type {vec_type.value}")
+            return ActionResult(notifs=[notif])
+
+        res = tree.query(vector_datas)
+        index = {r['sha1']: r['dist'] for r in res if r['sha1'] not in ignore_sha1s}
+
+        res_sha1s = list(index.keys())
+        res_scores = ScoreList(min=0, max=1, values=[index[sha1] for sha1 in res_sha1s],
+                               max_is_best=True,
+                               description="Similarity between 0 and 1. 1 is best")
+
+        res = Group(sha1s=res_sha1s, scores=res_scores)
+        return ActionResult(groups=[res])
+
+    async def search_by_text(self, context: ActionContext, vec_type: VectorType = VectorType.rgb, text: str = '', min_similarity: float = 0.5):
+        """Search image using text similarity"""
+        if text == '':
+            notif = Notif(type=NotifType.ERROR, name="EmptySearchText",
+                          message="Please give a valid and not empty text search argument")
+            return ActionResult(notifs=[notif])
+
+        context_instances = await self.project.get_instances(context.instance_ids)
+        context_sha1s = [i.sha1 for i in context_instances]
+
+        tree = await self._get_tree(vec_type)
+        if not tree:
+            notif = Notif(type=NotifType.ERROR, name="NoFaissTree",
+                          message=f"No Faiss tree could be loaded for vec_type {vec_type.value}")
+            return ActionResult(notifs=[notif])
+
+        if is_image_url(text):
+            im = Image.open(requests.get(text, stream=True).raw)
+            vec = self.transformer.to_vector(im)
+            resulting_images = tree.query([vec])
+        else:
+            try:
+                resulting_images = tree.query_texts([text], self.transformer)
+            except ValueError as e:
+                return ActionResult(notifs=[Notif(type=NotifType.ERROR, name="TextSimilarityError", message=str(e))])
+
+
+        # filter out images if they are not in the current context
+        filtered_instances = [inst for inst in resulting_images if inst['sha1'] in context_sha1s]
+
+        index = {r['sha1']: r['dist'] for r in filtered_instances}
+        res_sha1s = np.asarray(list(index.keys()))
+        res_scores = np.asarray([index[sha1] for sha1 in res_sha1s])
+
+        # remap score since text to image similary tends to be between 0.1 and 0.4 and filter by similarity
+        remaped_scores = np.around(np.interp(res_scores, [0, 0.375], [0, 1]), decimals=2)
+        final_scores = remaped_scores[remaped_scores >= min_similarity].tolist()
+        final_sha1s = res_sha1s[remaped_scores >= min_similarity].tolist()
+
+        scores = ScoreList(min=0, max=1, values=final_scores, description="Similarity between image and text never give less than 0.1 and more than 0.4, hence here the values, remapped between 0 and 1")
+        res = Group(sha1s=final_sha1s, scores=scores)
+        res.name = "Text Search: " + text
+        return ActionResult(groups=[res])
+
+    async def cluster_by_tags(self, context: ActionContext, tags: PropertyId, vec_type: VectorType = VectorType.rgb):
+        """Cluster images using a Tag/MultiTag property to guide the result"""
+        props = await self.project.get_properties(ids=[tags])
+        tag_prop = props[0]
+
+        if tag_prop.type != PropertyType.tag and tag_prop.type != PropertyType.multi_tags:
+            notif = Notif(type=NotifType.ERROR,
+                          name="WrongPropertyType",
+                          message=f"""Property: <{tag_prop.name}> is not of type Tag or MultiTags. This function only
+                                  accepts tag types properties. Please choose another property""")
+            return ActionResult(notifs=[notif])
+
+        instances = await self.project.get_instances(context.instance_ids)
+        sha1_to_instance = group_by_sha1(instances)
+        sha1s = list(sha1_to_instance.keys())
+        if not sha1s:
+            return None
+        # TODO: get tags text from the PropertyId
+        tags_text = [t.value for t in await self.project.get_tags(property_ids=[tags])]
+        text_vectors = get_text_vectors(tags_text, self.transformer)
+        pano_vectors = await self.project.get_vectors(source=self.name, vector_type=vec_type.value, sha1s=sha1s)
+
+        if not pano_vectors:
+            return ActionResult(notifs=[Notif(
+                NotifType.ERROR,
+                name="NoData",
+                message=f"""The Cluster_By_Tags function needs image vectors.
+                            No such vectors ({vec_type.value}) could be found. 
+                            Compute the vectors and try again.) """,
+                functions=self._get_vector_func_notifs(vec_type))])
+
+        groups = cluster_by_text(pano_vectors, text_vectors, tags_text)
+
+        return ActionResult(groups=groups)
+
+    async def find_duplicates(self, context: ActionContext, min_similarity: float = 0.95):
+        """
+        Create clusters with at least `min_similarity` between the images of the cluster
+        @min_similarity: the minimal similarity value between images of the cluster
+        """
+        # on récupère les vecteurs
+        # pour chaque vecteur on récupère ses plus similaires (150 pour test) puis on filtre tout ce qui est < min_similarity
+        # on marque tous les images dans le cluster pour ne pas les requêter à nouveau
+        instances = await self.project.get_instances(context.instance_ids)
+        sha1_to_instance = group_by_sha1(instances)
+        sha1s = list(sha1_to_instance.keys())
+        if not sha1s:
+            return None
+        # TODO: get tags text from the PropertyId
+        pano_vectors = await self.project.get_vectors(source=self.name, vector_type=VectorType.rgb.value, sha1s=sha1s)
+        vectors, sha1s = zip(*[(i.data, i.sha1) for i in pano_vectors])
+        already_in_clusters = set()
+        groups = []
+        for vector, sha1 in zip(vectors, sha1s):
+            if sha1 in already_in_clusters:
+                continue
+            tree = await self._get_tree(VectorType.rgb)
+            res = tree.query([vector.data], 150)
+            filtered = [r for r in res if r['dist'] >= min_similarity and r['sha1'] in sha1s]
+            res_sha1s = [r['sha1'] for r in filtered]
+            res_scores = [r['dist'] for r in filtered]
+            score_list = ScoreList(min=0, max=1, max_is_best=True, values=res_scores)
+            if len(res_sha1s) == 1:
+                continue
+            already_in_clusters.update(res_sha1s)
+            groups.append(Group(sha1s=res_sha1s, scores=score_list))
+        return ActionResult(groups=groups)
+
+    async def rebuild_trees(self, deleted):
+        for type_ in VectorType:
+            await self._update_tree(type_)
+
+    async def _get_tree(self, vec_type: VectorType):
+        tree = self.trees.get(vec_type)
+        if tree:
+            return tree
+        tree = load_faiss_tree(self, vec_type)
+        if tree:
+            self.trees[vec_type] = tree
+            return tree
+        tree = await create_faiss_tree(self, vec_type)
+        if tree:
+            self.trees[vec_type] = tree
+            return tree
+
+    async def _update_tree(self, vec_type: VectorType):
+        tree = await create_faiss_tree(self, vec_type)
+        self.trees[vec_type] = tree
+        print(f"updated {vec_type.value} faiss tree")
+        return tree
+
+    def _load_transformer(self):
+        if TransformerName[self.params.model] == TransformerName.auto and not self.params.hugging_face_model:
+            return ActionResult(notifs=[Notif(
+                NotifType.ERROR,
+                name="No hugging face model specified",
+                message=f"""
+                Automodel selected but no hugging face model provided
+                """)])
+        return get_transformer(TransformerName[self.params.model], self.params.hugging_face_model)

panopticml/utils.py

@@ -0,0 +1,72 @@
+import io
+
+import numpy as np
+import torch
+import re
+from PIL import Image
+
+from .models import VectorType
+
+
+def preprocess_image(image_data: bytes, vector_type: VectorType = VectorType.rgb):
+    image = Image.open(io.BytesIO(image_data))
+    if vector_type == VectorType.rgb:
+        image = image.convert('RGB')
+    else:
+        image = image.convert('L').convert('RGB')
+    return image
+
+def cosine_similarity(embedding1: np.array, embedding2: np.array):
+    """
+    Calcule la similarité cosinus entre deux embeddings normalisés
+
+    Args:
+        embedding1: premier embedding (normalisé)
+        embedding2: deuxième embedding (normalisé)
+
+    Returns:
+        float: similarité cosinus (entre -1 et 1)
+    """
+    # Pour des embeddings normalisés, similarité cosinus = produit scalaire
+    embedding1_torch = torch.from_numpy(embedding1).squeeze()
+    embedding2_torch = torch.from_numpy(embedding2).squeeze()
+
+    return torch.dot(embedding1_torch, embedding2_torch).item()
+
+
+def similarity_matrix(vectors1: list[np.array], vectors2: list[np.array]) -> (torch.Tensor, torch.Tensor):
+    """
+    Create a similarity matrix from two list of normalized np vectors
+    Returns two tensors, one for best scores and one for best_indices of vectors1 into vectors2
+    i.e for each vector1 we'll know the closest vector of vectors2 and the similarity between them
+    """
+    astorch1 = torch.from_numpy(np.vstack(vectors1))
+    astorch2 = torch.from_numpy(np.vstack(vectors2))
+    matrix = torch.mm(astorch1, astorch2.T)
+    best_scores, best_indices = torch.max(matrix, dim=1)
+    return best_scores, best_indices
+
+def resolve_device():
+    device = 'cpu'
+    if torch.cuda.is_available():
+        device = 'cuda'
+    # TODO: when silicon bugs are working again put it back
+    # elif torch.backends.mps.is_available():
+    #     device = 'mps'
+    return device
+
+def is_image_url(url):
+    pattern = re.compile(
+        r'''(?xi)
+        \b
+        https?://                       # protocole http ou https
+        [\w.-]+(?:\.[\w.-]+)+           # domaine
+        (?:/[^\s?#]*)*                  # chemin éventuel
+        /?                              # éventuellement un / final
+        [^\s?#]*                        # éventuellement un nom de fichier
+        \.(?:jpg|jpeg|png|gif|webp|svg|bmp|tiff|ico)  # extension image
+        (?:\?[^\s#]*)?                  # paramètres après ?
+        \b
+        '''
+    )
+    return re.match(pattern, url)

panopticml-0.0.2.dist-info/METADATA

@@ -0,0 +1,55 @@
+Metadata-Version: 2.4
+Name: panopticml
+Version: 0.0.2
+Summary: A package for Panoptic ML
+Project-URL: Homepage, https://github.com/CERES-Sorbonne/PanopticML
+Project-URL: Bug Tracker, https://github.com/CERES-Sorbonne/PanopticML/issues
+Author-email: Félix Alié <felix.alie@sorbonne-universite.fr>
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Requires-Python: >=3.10
+Requires-Dist: faiss-cpu<1.11.0
+Requires-Dist: hdbscan
+Requires-Dist: mistralai
+Requires-Dist: numpy; platform_system != 'Darwin'
+Requires-Dist: numpy<=1.26.0; platform_system == 'Darwin'
+Requires-Dist: scikit-learn; platform_system != 'Darwin'
+Requires-Dist: scikit-learn==1.5.0; platform_system == 'Darwin'
+Requires-Dist: scipy
+Requires-Dist: torch; platform_system != 'Darwin'
+Requires-Dist: torch==2.1.0; platform_system == 'Darwin'
+Requires-Dist: torchvision
+Requires-Dist: transformers<4.52.0
+Description-Content-Type: text/markdown
+
+# PanopticML
+
+Default plugin for panoptic, provides all machine learning functionalities such as:
+- image embeddings using OpenAI CLIP
+- images clustering using FAISS Kmeans or HDBScan
+- images to images similarity using FAISS Index L2
+- text to images similarity using FAISS Index L2 + CLIP
+
+## Available models
+
+Several models are available in this plugin to compute the embeddings: 
+- [google mobilenet](https://huggingface.co/google/mobilenet_v2_1.4_224): a really light model with lower performances but made to run on bad devices, no support for text similarity
+- [openAI CLIP](https://huggingface.co/openai/clip-vit-large-patch14): default model for panoptic, good performances, runs well on most computers with a decent CPU, support text similarity and has a good understanding of semantics
+- [meta dinosV2](https://huggingface.co/docs/transformers/en/model_doc/dinov2): model that will have better performances than CLIP on pure visual similarities but less understanding of themes and semantics, no support for text similarity, runs well on CPU
+- [google SIGLIP2](https://huggingface.co/docs/transformers/main/model_doc/siglip2): computing heavy model, NVDIA GPU recommended but works also on CPU (it will take a long time to compute embeddings), way better results than CLIP on text similarity and semantics, and pretty good visual features
+- auto transformers: want to tryout any huggingface multimodal model ? you can just provide its id to panopticML and should be able to use it directly
+
+## Clustering functions
+- Kmeans: specify a number of clusters, really fast
+- HDBScan: automatic number of clusters, a bit slower and excludes a lot of images
+- Images to text: considering several texts, create a cluster per text and put each image to the text it is the closest to
+- Find duplicates: create clusters with only images sharing a lot of similarity together, good to find duplicates or near duplicates

panopticml-0.0.2.dist-info/RECORD

@@ -0,0 +1,13 @@
+panopticml/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+panopticml/compute_vector_task.py,sha256=Fe2CRez6tXReJ46v0xREJRKY3RYm4iaIx-maUpr_FQE,1957
+panopticml/models.py,sha256=JSex5wzu1S0iw1xnGJN1zdHo7ARnxXy2jLhHZ-mCQVs,92
+panopticml/panoptic_ml.py,sha256=XH8mSLfcSBvpnBNtqrDenPBl-6FSomke-rTHBL4zyT0,16047
+panopticml/utils.py,sha256=28qUMoBh-qaI8-Y8fIy0rFr3evTV6J1kr5dazehedcI,2470
+panopticml/compute/__init__.py,sha256=Y5okbGIQHPYCal9curiu1-WFES29Qd2Lc9RY2V-BKBI,37
+panopticml/compute/clustering.py,sha256=O1Yshea9D7PCqDE_bAMwKyBkAyPei5Wz_pS2xhayD-E,4937
+panopticml/compute/faiss_tree.py,sha256=WY2oZJCiGMWbyVWIqaNEgsW16VXH5tt17Zf9P5vrpZ4,2408
+panopticml/compute/similarity.py,sha256=u4GmoaWQqq-fBqChiHl6HsLfK8QqVZasIscbbzB9I7c,364
+panopticml/compute/transformers/__init__.py,sha256=zCVjpY1E_afMbmB2bWLT1grPbOlqj7MsxDcDR1sOHec,4381
+panopticml-0.0.2.dist-info/METADATA,sha256=XClZCxAWyjGZerkz3iyoeeCRPLiH9lrHZQj8cWqlGuc,3381
+panopticml-0.0.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+panopticml-0.0.2.dist-info/RECORD,,

panopticml-0.0.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any