panoptic 0.0.9.dev9__tar.gz → 0.1.0__tar.gz

{panoptic-0.0.9.dev9 → panoptic-0.1.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: panoptic
-Version: 0.0.9.dev9
-License: Mozilla
+Version: 0.1.0
+License: MPL-2.0 license
 Description-Content-Type: text/markdown
 
 # Panoptic

{panoptic-0.0.9.dev9 → panoptic-0.1.0}/panoptic/api.py

@@ -1,25 +1,27 @@
+import logging
 import os
 from sys import platform
 from typing import Optional
 
 import aiofiles as aiofiles
+import pandas as pd
 from fastapi import FastAPI, Query, UploadFile
 from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import ORJSONResponse
 from pydantic import BaseModel
 from starlette.responses import Response
 from starlette.staticfiles import StaticFiles
-import pandas as pd
 
 from panoptic import core
-from panoptic.scripts.to_pca import compute_all_pca
-from panoptic.core import create_property, add_property_to_images, get_images, create_tag, \
-    delete_image_property, \
+from panoptic.core import create_property, create_tag, \
     update_tag, get_tags, get_properties, delete_property, update_property, delete_tag, delete_tag_parent, add_folder, \
-    db_utils, make_clusters, get_similar_images, get_full_image, read_properties_file
+    db_utils, make_clusters, get_similar_images, read_properties_file, get_full_images, set_property_values
 from panoptic.core import db
-from panoptic.models import Property, Images, Tag, Tags, Properties, PropertyPayload, \
-    AddImagePropertyPayload, AddTagPayload, DeleteImagePropertyPayload, \
-    UpdateTagPayload, UpdatePropertyPayload, Tab, MakeClusterPayload
+from panoptic.models import Property, Tag, Tags, Properties, PropertyPayload, \
+    SetPropertyValuePayload, AddTagPayload, DeleteImagePropertyPayload, \
+    UpdateTagPayload, UpdatePropertyPayload, Tab, MakeClusterPayload, PropertyValue, GetSimilarImagesPayload, \
+    ChangeProjectPayload
+from panoptic.scripts.to_pca import compute_all_pca
 
 app = FastAPI()
 app.add_middleware(
@@ -42,7 +44,7 @@ async def startup_event():
 # Route pour créer une property et l'insérer dans la table des properties
 @app.post("/property")
 async def create_property_route(payload: PropertyPayload) -> Property:
-    return await create_property(payload.name, payload.type)
+    return await create_property(payload.name, payload.type, payload.mode)
 
 
 @app.get("/property")
@@ -54,22 +56,24 @@ async def get_properties_route() -> Properties:
 async def update_property_route(payload: UpdatePropertyPayload) -> Property:
     return await update_property(payload)
 
+
 @app.post('/property/file')
 async def properties_by_file(file: UploadFile):
     data = pd.read_csv(file.file, sep=";")
     return await read_properties_file(data)
 
+
 @app.delete('/property/{property_id}')
 async def delete_property_route(property_id: str):
     await delete_property(property_id)
-    return f"Property {property_id} correctly deleted"
+    return await get_properties()
 
 
 # Route pour récupérer la liste de toutes les images
-@app.get("/images")
+@app.get("/images", response_class=ORJSONResponse)
 async def get_images_route():
-    images = await get_images()
-    return images
+    images = await get_full_images()
+    return ORJSONResponse(images)
 
 
 @app.get('/images/{file_path:path}')
@@ -90,15 +94,15 @@ async def get_image(file_path: str):
 # Route pour ajouter une property à une image dans la table de jointure entre image et property
 # On retourne le payload pour pouvoir valider l'update côté front
 @app.post("/image_property")
-async def add_image_property(payload: AddImagePropertyPayload) -> AddImagePropertyPayload:
-    await add_property_to_images(payload.property_id, payload.sha1_list, payload.value)
-    return payload
+async def add_image_property(payload: SetPropertyValuePayload):
+    updated, value = await set_property_values(property_id=payload.property_id, image_ids=payload.image_ids,  sha1s=payload.sha1s, value=payload.value)
+    return {'updated_ids': updated, 'value': value}
 
 
 # Route pour supprimer une property d'une image dans la table de jointure entre image et property
 @app.delete("/image_property")
-async def delete_property_route(payload: DeleteImagePropertyPayload) -> DeleteImagePropertyPayload:
-    await delete_image_property(payload.property_id, payload.sha1)
+async def delete_property_value_route(payload: DeleteImagePropertyPayload) -> DeleteImagePropertyPayload:
+    await db.delete_property_value(payload.property_id, image_ids=[payload.image_id])
     return payload
 
 
@@ -109,10 +113,10 @@ async def add_tag(payload: AddTagPayload) -> Tag:
     return await create_tag(payload.property_id, payload.value, payload.parent_id, payload.color)
 
 
-@app.get("/tags")
-async def get_tags_route(property: Optional[str] = None) -> Tags:
-    return await get_tags(property)
-
+@app.get("/tags", response_class=ORJSONResponse)
+async def get_tags_route(property: Optional[str] = None):
+    tags = await get_tags(property)
+    return ORJSONResponse(tags)
 
 @app.patch("/tags")
 async def update_tag_route(payload: UpdateTagPayload) -> Tag:
@@ -139,9 +143,18 @@ async def get_folders_route():
 @app.get('/import_status')
 async def get_import_status_route():
     image_import = core.importer
-    new_image = core.get_new_images()
-    update = [await get_full_image(i) for i in new_image]
-    res = {'to_import': image_import.total_import, 'imported': image_import.current_computed, 'new_images': update}
+    new_image = image_import.get_new_images()
+
+    update = []
+    if new_image:
+        update = await get_full_images(new_image)
+
+    res = {
+        'to_import': image_import.total_import,
+        'imported': image_import.current_import,
+        'computed': image_import.current_computed,
+        'new_images': update
+    }
     return res
 
 
@@ -181,12 +194,41 @@ async def make_clusters_route(payload: MakeClusterPayload) -> list[list[str]]:
     return await make_clusters(payload.nb_groups, payload.image_list)
 
 
-@app.get("/similar/")
-async def get_similar_images_route(sha1_list: list[str] = Query(None)) -> list:
-    return await get_similar_images(sha1_list)
+@app.post("/similar")
+async def get_similar_images_route(payload: GetSimilarImagesPayload) -> list:
+    return await get_similar_images(payload.sha1_list)
+
 
 @app.post("/pca")
 async def start_pca_route():
     return await compute_all_pca(force=True)
 
+
+@app.post("/project")
+async def change_project_route(payload: ChangeProjectPayload):
+    os.environ['PANOPTIC_DATA'] = payload.project
+    await db_utils.init()
+    return f"changed project to {payload.project}"
+
+@app.get('/small/images/{file_path:path}')
+async def get_image(file_path: str):
+    path = os.path.join(os.environ['PANOPTIC_DATA'], 'mini', file_path)
+    async with aiofiles.open(path, 'rb') as f:
+        data = await f.read()
+
+    ext = path.split('.')[-1]
+
+    # # media_type here sets the media type of the actual response sent to the client.
+    return Response(content=data, media_type="image/" + ext)
+
+# app.mount("/small/images/", StaticFiles(directory=os.path.join('PANOPTIC_DATA', 'mini')), name="static")
 app.mount("/", StaticFiles(directory=os.path.join(os.path.dirname(__file__), "html"), html=True), name="static")
+
+
+class EndpointFilter(logging.Filter):
+    def filter(self, record: logging.LogRecord) -> bool:
+        return record.getMessage().find("/import_status") == -1
+
+
+# Filter out /endpoint
+logging.getLogger("uvicorn.access").addFilter(EndpointFilter())

{panoptic-0.0.9.dev9 → panoptic-0.1.0}/panoptic/compute/__init__.py

@@ -1,2 +1,2 @@
-from .transform import transform_image, transform_directory, to_average_hash, to_sha1, to_vector, can_use_pca, create_pca, to_pca
+from .transform import transform_image, transform_directory, to_average_hash, to_sha1, to_vector, can_compute_pca, create_pca, to_pca
 from .similarity import make_clusters, get_similar_images, create_similarity_tree

{panoptic-0.0.9.dev9 → panoptic-0.1.0}/panoptic/compute/similarity.py

@@ -5,12 +5,12 @@ import numpy as np
 from sklearn.cluster import DBSCAN, KMeans, estimate_bandwidth, MeanShift
 
 from panoptic.compute.utils import load_similarity_tree, SimilarityTreeWithLabel
-from panoptic.models import ImageVector
+from panoptic.models import ComputedValue
 
 SIMILARITY_TREE: SimilarityTreeWithLabel = load_similarity_tree()
 
 
-def create_similarity_tree(images: list[ImageVector]):
+def create_similarity_tree(images: list[ComputedValue]):
     tree = SimilarityTreeWithLabel(images)
     with open(os.path.join(os.getenv('PANOPTIC_DATA'), 'tree.pkl'), 'wb') as f:
         pickle.dump(tree, f)
@@ -25,7 +25,7 @@ def get_similar_images(vectors: list[np.ndarray]):
     return SIMILARITY_TREE.query(vector)
 
 
-def make_clusters(images: list[ImageVector], *, method='kmeans', **kwargs) -> list[list[str]]:
+def make_clusters(images: list[ComputedValue], *, method='kmeans', **kwargs) -> list[list[str]]:
     res = []
     vectors, sha1, ahashs = zip(*[(i.vector, i.sha1, i.ahash) for i in images])
     sha1 = np.asarray(sha1)

{panoptic-0.0.9.dev9 → panoptic-0.1.0}/panoptic/compute/transform.py

@@ -13,7 +13,7 @@ import glob
 from PIL import Image
 from sklearn.decomposition import PCA
 from sklearn.neighbors import KDTree
-from transformers import AutoImageProcessor, MobileNetV2Model, CLIPVisionModel, CLIPProcessor
+from transformers import AutoImageProcessor, MobileNetV2Model
 # from .ocr import full_ocr
 from .utils import load_data
 
@@ -89,7 +89,7 @@ def transform_image(_image_path: str, ocr=False):
 
     data[sha1] = result
     # can we trigger pca and use it ?
-    if pca is None and can_use_pca(len(data.values()), list(data.values())[0]['vector']):
+    if pca is None and can_compute_pca(len(data.values()), list(data.values())[0]['vector']):
         sha1_list = list(data.keys())
         vector_list = [data[sha1_key]['vector'] for sha1_key in sha1_list]
         create_pca(vector_list)
@@ -159,7 +159,7 @@ def save_pca():
         pickle.dump(pca, f)
 
 
-def can_use_pca(nb_vectors: int, vector_sample: np.array) -> bool:
+def can_compute_pca(nb_vectors: int, vector_sample: np.array) -> bool:
     """
     Can we start training a PCA to reduce vector size ?
     Basically it's: do we have more vectors than 110% the number of dimensions ?
@@ -169,7 +169,7 @@ def can_use_pca(nb_vectors: int, vector_sample: np.array) -> bool:
     """
     if not USE_PCA_IF_POSSIBLE:
         return False
-    if pca is not None:
+    if get_pca() is not None:
         return False
     if nb_vectors > vector_sample.shape[0] + (vector_sample.shape[0] / 10):
         return True

{panoptic-0.0.9.dev9 → panoptic-0.1.0}/panoptic/compute/utils.py

@@ -4,10 +4,11 @@ import pickle
 import numpy as np
 from sklearn.neighbors import KDTree
 
-from panoptic.models import ImageVector
+from panoptic.models import ComputedValue
 
 os.environ['PANOPTIC_DATA'] = os.getenv('PANOPTIC_DATA', os.getcwd())
 
+
 def load_data(data_path=os.getenv('PANOPTIC_DATA')):
     if not data_path:
         raise ValueError("PANOPTIC_DATA was not set")
@@ -42,7 +43,7 @@ def load_similarity_tree():
 
 
 class SimilarityTreeWithLabel:
-    def __init__(self, images: list[ImageVector]):
+    def __init__(self, images: list[ComputedValue]):
         vectors, sha1_list = zip(*[(i.vector, i.sha1) for i in images])
         self.image_labels = sha1_list
         self.tree = KDTree(vectors)

panoptic-0.1.0/panoptic/core/__init__.py

@@ -0,0 +1,266 @@
+import atexit
+import json
+import logging
+import random
+from concurrent.futures import ProcessPoolExecutor
+import time
+from typing import List, Any
+
+import pandas
+from fastapi import HTTPException
+from tqdm import tqdm
+
+from panoptic.core import db
+from panoptic import compute
+from panoptic.models import PropertyType, JSON, Tag, Property, Tags, Properties, \
+    UpdateTagPayload, UpdatePropertyPayload, Image, PropertyValue
+from .image_importer import ImageImporter
+from .image_importer import ImageImporter
+
+executor = ProcessPoolExecutor(max_workers=4)
+atexit.register(executor.shutdown)
+importer = ImageImporter(executor)
+
+
+async def create_property(name: str, property_type: PropertyType, mode='id') -> Property:
+    return await db.add_property(name, property_type.value, mode)
+
+
+async def update_property(payload: UpdatePropertyPayload) -> Property:
+    existing_property = await db.get_property_by_id(payload.id)
+    if not existing_property:
+        raise HTTPException(status_code=400, detail="Trying to modify non existent property")
+    new_property = existing_property.copy(update=payload.dict(exclude_unset=True))
+    await db.update_property(new_property)
+    return new_property
+
+
+async def get_properties() -> Properties:
+    properties = await db.get_properties()
+    return {prop.id: prop for prop in properties}
+
+
+async def delete_property(property_id: str):
+    await db.delete_property(property_id)
+
+
+async def set_property_values(property_id: int, value: Any, image_ids: List[int] = None, sha1s: List[int] = None):
+    if image_ids and sha1s:
+        raise TypeError('Only image_ids or sha1s should be given as keys. Never both')
+
+    prop = await db.get_property_by_id(property_id)
+    if prop.mode == 'id' and not image_ids:
+        raise TypeError(f'Property {property_id}: {prop.name} needs image ids as key [mode: {prop.mode}]')
+    if prop.mode == 'sha1' and not sha1s:
+        raise TypeError(f'Property {property_id}: {prop.name} needs sha1s as key [mode: {prop.mode}]')
+
+    if prop.type == PropertyType.tag or prop.type == PropertyType.multi_tags:
+        if value and not isinstance(value, list):
+            value = [int(value)]
+
+    return await db.set_property_values(property_id, value, image_ids, sha1s)
+
+
+async def get_full_images(image_ids: List[int] = None) -> List[Image]:
+    images = await db.get_images(image_ids)
+    sha1s = list({img.sha1 for img in images})
+    property_values = await db.get_property_values(image_ids=image_ids)
+    ahashs = await db.get_sha1_ahashs(sha1s=sha1s)
+    image_index = {img.id: img for img in images}
+
+    def assign_value(prop):
+        image_index[prop.image_id].properties[prop.property_id] = prop
+
+    [assign_value(prop) for prop in property_values if prop.image_id >= 0]
+
+    sha1_properties = {}
+
+    def register_sha1_value(prop: PropertyValue):
+        if prop.sha1 not in sha1_properties:
+            sha1_properties[prop.sha1] = []
+        sha1_properties[prop.sha1].append(prop)
+
+    [register_sha1_value(prop) for prop in property_values if prop.image_id < 0]
+
+    def assign_sha1_value(image_id, prop: PropertyValue):
+        image_index[image_id].properties[prop.property_id] = prop
+
+    [assign_sha1_value(img.id, prop) for img in images if img.sha1 in sha1_properties for prop in
+     sha1_properties[img.sha1]]
+
+    [setattr(img, 'ahash', ahashs[img.sha1]) for img in images if img.sha1 in ahashs]
+    return images
+
+
+async def make_clusters(sensibility: float, sha1s: [str]) -> list[list[str]]:
+    """
+    Compute clusters and return a list of lists of sha1
+    """
+    if not sha1s:
+        return []
+    values = await db.get_sha1_computed_values(sha1s)
+    return compute.make_clusters(values, method="kmeans", nb_clusters=sensibility)
+
+
+async def get_similar_images(sha1s: list[str]):
+    vectors = [i.vector for i in await db.get_sha1_computed_values(sha1s)]
+    res = compute.get_similar_images(vectors)
+    return [img for img in res if img['sha1'] not in sha1s]
+
+
+async def add_property_to_images(property_id: int, sha1_list: list[str], value: JSON) -> str:
+    # first check that the property and the image exist:
+    images = await db.get_images(sha1s=sha1_list)
+    if await db.get_property_by_id(property_id) and images:
+        await db.set_property_values(image_ids=[img.id for img in images], property_id=property_id, value=value)
+        # check if a value already exists
+        # if await db.get_image_property(sha1, property_id):
+        #     await db.update_image_property(sha1, property_id, value)
+        # else:
+        #     await db.add_image_property(sha1, property_id, value)
+        return value
+    else:
+        raise HTTPException(status_code=400, detail="Trying to set a value on a non existent property or sha1")
+
+
+async def read_properties_file(data: pandas.DataFrame):
+    filenames, ids, sha1 = zip(*[(i.name, i.id, i.sha1) for i in await db.get_images()])
+
+    # if there are no duplicates them lets assume the props are sha1
+    if data.duplicated('key').sum() == 0:
+        prop_mode = "sha1"
+        matcher = dict(zip(filenames, sha1))
+    else:
+        prop_mode = "id"
+        matcher = dict(zip(filenames, ids))
+    data = data[data.key.isin(filenames)]
+    # data = data.drop_duplicates(subset='key', keep='first')
+    # add panoptic id or sha1 to the dataframe
+    data.loc[data.index, 'panoptic_id'] = data['key'].map(matcher)
+
+    # first, create new images where ids are the same
+    unique_ids = list(data.panoptic_id.unique())
+    clones_created = 0
+    # TODO: create clones all at once ? it's tricky with the ids to update
+    if prop_mode == "id":
+        for id in tqdm(unique_ids):
+            sub_data = data[data.panoptic_id == id]
+            # create clones only if needed
+            if len(list(sub_data.panoptic_id)) == 0:
+                continue
+            image = await db.get_images([id])
+            image = image[0]
+            new_ids = await db.create_clones(image, sub_data.shape[0] - 1)
+            clones_created += len(new_ids)
+            data.loc[data.panoptic_id == id, "panoptic_id"] = [image.id, *new_ids]
+        logging.getLogger('panoptic').info(f"created {clones_created} new images")
+    properties_to_create = data.columns.tolist()
+
+    # then for each property to create
+    for prop in properties_to_create:
+        if prop == "key" or prop == "panoptic_id":
+            continue
+
+        # create the property
+        prop_name, prop_type = prop.split('[')
+        prop_type = PropertyType(prop_type.split(']')[0])
+        property = await create_property(prop_name, prop_type, prop_mode)
+        # then get all possible values to insert
+        prop_values = list(data[prop].unique())
+
+        # if it's a tag property let's create the tags in the db
+        if property.type == PropertyType.tag or property.type == PropertyType.multi_tags:
+            tag_matcher = {}
+            # TODO: can we optimize to create all tags at once ?
+            for value in tqdm(prop_values):
+                created_tags = []
+                # if it's multi tag, assume tags are separated by a comma and create them separately
+                for single_tag in value.split(','):
+                    colors = ["7c1314", "c31d20", "f94144", "f3722c", "f8961e", "f9c74f", "90be6d", "43aa8b", "577590",
+                              "9daebe"]
+                    color = '#' + colors[random.randint(0, len(colors) - 1)]
+                    tag = await create_tag(property.id, single_tag, 0, color)
+                    created_tags.append(tag.id)
+                tag_matcher[value] = created_tags
+            # now change all values in the dataframe with the real value that we are going to insert
+            data.loc[data.index, prop] = data[prop].map(tag_matcher)
+        await db.set_multiple_property_values(property.id, list(data[prop]), list(data.panoptic_id))
+
+
+async def add_folder(folder):
+    found = await importer.import_folder(folder)
+    print(f'found {found} images')
+    return folder
+
+
+new_images = []
+
+
+def get_new_images():
+    copy = [i for i in new_images]
+    new_images.clear()
+    return copy
+
+
+async def create_tag(property_id, value, parent_id, color: str) -> Tag:
+    existing_tag = await db.get_tag(property_id, value)
+    if existing_tag is not None:
+        if await db.tag_in_ancestors(existing_tag.id, parent_id):
+            raise HTTPException(status_code=400, detail="Adding a tag that is an ancestor of himself")
+        existing_tag.parents = list({*existing_tag.parents, parent_id})
+        await db.update_tag(existing_tag)
+        tag_id = existing_tag.id
+    else:
+        parents = [parent_id]
+        tag_id = await db.add_tag(property_id, value, json.dumps(parents), color)
+    return await db.get_tag_by_id(tag_id)
+
+
+async def update_tag(payload: UpdateTagPayload) -> Tag:
+    existing_tag = await db.get_tag_by_id(payload.id)
+    if not existing_tag:
+        raise HTTPException(status_code=400, detail="Trying to modify non existent tag")
+    if await db.tag_in_ancestors(existing_tag.id, payload.parent_id):
+        raise HTTPException(status_code=400, detail="Adding a tag that is an ancestor of himself")
+    # change only fields of the tags that are set in the payload
+    new_tag = existing_tag.copy(update=payload.dict(exclude_unset=True))
+    await db.update_tag(new_tag)
+    return new_tag
+
+
+async def delete_tag(tag_id: int) -> List[int]:
+    # first delete the tag
+    modified_tags = [await db.delete_tag_by_id(tag_id)]
+    # when deleting a tag, get all children of this tag
+    children = await db.get_tags_by_parent_id(tag_id)
+    # and remove parent ref of tag in children
+    [modified_tags.extend(await delete_tag_parent(c.id, tag_id)) for c in children]
+    # then get all images properties tagged with it
+    property_values = await db.get_property_values_with_tag(tag_id)
+    # and delete the tag ref from them
+    for data in property_values:
+        if tag_id in data.value:
+            data.value.remove(tag_id)
+            await db.set_property_values(data.property_id, data.value, [data.image_id])
+    return modified_tags
+
+
+async def delete_tag_parent(tag_id: int, parent_id: int) -> List[int]:
+    print(tag_id, parent_id)
+    tag = await db.get_tag_by_id(tag_id)
+    tag.parents.remove(parent_id)
+    if not tag.parents:
+        return await delete_tag(tag.id)
+    else:
+        await db.update_tag(tag)
+        return []
+
+
+async def get_tags(prop: str = None) -> Tags:
+    res = {}
+    tag_list = await db.get_tags(prop)
+    for tag in tag_list:
+        if tag.property_id not in res:
+            res[tag.property_id] = {}
+        res[tag.property_id][tag.id] = tag
+    return res