geovisio 2.8.1__py3-none-any.whl → 2.9.0__py3-none-any.whl

geovisio/web/items.py

@@ -6,13 +6,16 @@ from typing import Dict, List, Optional, Any
 from urllib.parse import unquote
 from psycopg.types.json import Jsonb
 from pydantic import BaseModel, ConfigDict, ValidationError, field_validator, model_validator
+from shapely import intersects
 from werkzeug.datastructures import MultiDict
 from uuid import UUID
 from geovisio import errors, utils
 from geovisio.utils import auth, db
+from geovisio.utils.cql2 import parse_search_filter
 from geovisio.utils.params import validation_error
 from geovisio.utils.pictures import cleanupExif
-from geovisio.utils.semantics import SemanticTagUpdate, Entity, EntityType, update_tags
+from geovisio.utils.semantics import Entity, EntityType, update_tags
+from geovisio.utils.tags import SemanticTagUpdate
 from geovisio.web.params import (
     as_latitude,
     as_longitude,
@@ -138,6 +141,7 @@ def dbPictureToStacItem(seqId, dbPic):
                     "pers:roll": dbPic["metadata"].get("roll"),
                     "geovisio:status": dbPic.get("status"),
                     "geovisio:producer": dbPic["account_name"],
+                    "geovisio:rank_in_collection": dbPic["rank"],
                     "original_file:size": dbPic["metadata"].get("originalFileSize"),
                     "original_file:name": dbPic["metadata"].get("originalFileName"),
                     "panoramax:horizontal_pixel_density": dbPic.get("h_pixel_density"),
@@ -145,7 +149,8 @@ def dbPictureToStacItem(seqId, dbPic):
                     "geovisio:thumbnail": _getThumbJpgPictureURL(dbPic["id"], dbPic.get("status")),
                     "exif": removeNoneInDict(cleanupExif(dbPic["exif"])),
                     "quality:horizontal_accuracy": float("{:.1f}".format(dbPic["gps_accuracy_m"])) if dbPic.get("gps_accuracy_m") else None,
-                    "semantics": dbPic["semantics"] if "semantics" in dbPic else None,
+                    "semantics": [s for s in dbPic.get("semantics") or [] if s],
+                    "annotations": [a for a in dbPic.get("annotations") or [] if a],
                 }
             ),
             "links": cleanNoneInList(
@@ -437,19 +442,12 @@ def getCollectionItems(collectionId):
                     CASE WHEN LAG(p.status) OVER othpics = 'ready' THEN ST_AsGeoJSON(LAG(p.geom) OVER othpics)::json END AS prevpicgeojson,
                     CASE WHEN LEAD(p.status) OVER othpics = 'ready' THEN LEAD(p.id) OVER othpics END AS nextpic,
                     CASE WHEN LEAD(p.status) OVER othpics = 'ready' THEN ST_AsGeoJSON(LEAD(p.geom) OVER othpics)::json END AS nextpicgeojson,
-                    t.semantics
+                    get_picture_semantics(p.id) as semantics,
+                    get_picture_annotations(p.id) as annotations
                 FROM sequences_pictures sp
                 JOIN pictures p ON sp.pic_id = p.id
                 JOIN accounts a ON a.id = p.account_id
                 JOIN sequences s ON s.id = sp.seq_id
-                LEFT JOIN (
-                    SELECT picture_id, json_agg(json_strip_nulls(json_build_object(
-                        'key', key,
-                        'value', value
-                    ))) AS semantics
-                    FROM pictures_semantics
-                    GROUP BY picture_id
-                ) t ON t.picture_id = p.id
                 WHERE
                     {filter}
                 WINDOW othpics AS (PARTITION BY sp.seq_id ORDER BY sp.rank)
@@ -460,15 +458,14 @@ def getCollectionItems(collectionId):
 
             records = cursor.execute(query, params)
 
-            bounds: Optional[Bounds] = None
             items = []
+            first_rank, last_rank = None, None
             for dbPic in records:
-                if not bounds:
-                    bounds = Bounds(min=dbPic["rank"], max=dbPic["rank"])
-                else:
-                    bounds.update(dbPic["rank"])
-
+                if first_rank is None:
+                    first_rank = dbPic["rank"]
+                last_rank = dbPic["rank"]
                 items.append(dbPictureToStacItem(collectionId, dbPic))
+            bounds = Bounds(first=first_rank, last=last_rank) if records else None
 
             links = [
                 get_root_link(),
@@ -491,8 +488,8 @@ def getCollectionItems(collectionId):
             ]
 
             if paginated and items and bounds:
-                if bounds.min:
-                    has_item_before = bounds.min > seqMeta["min_rank"]
+                if bounds.first:
+                    has_item_before = bounds.first > seqMeta["min_rank"]
                     if has_item_before:
                         links.append(
                             {
@@ -504,7 +501,7 @@ def getCollectionItems(collectionId):
                         # Previous page link
                         #   - If limit is set, rank is current - limit -1
                         #   - If no limit is set, rank is 0 (none)
-                        prevRank = bounds.min - limit - 1 if limit is not None else 0
+                        prevRank = bounds.first - limit - 1 if limit is not None else 0
                         if prevRank < 1:
                             prevRank = None
                         links.append(
@@ -521,7 +518,7 @@ def getCollectionItems(collectionId):
                             }
                         )
 
-                has_item_after = bounds.max < seqMeta["max_rank"]
+                has_item_after = bounds.last < seqMeta["max_rank"]
                 if has_item_after:
                     links.append(
                         {
@@ -532,7 +529,7 @@ def getCollectionItems(collectionId):
                                 _external=True,
                                 collectionId=collectionId,
                                 limit=limit,
-                                startAfterRank=bounds.max,
+                                startAfterRank=bounds.last,
                             ),
                         }
                     )
@@ -543,10 +540,10 @@ def getCollectionItems(collectionId):
 
                     lastPageRank = startAfterRank
                     if limit is not None:
-                        if seqMeta["max_rank"] > bounds.max:
+                        if seqMeta["max_rank"] > bounds.last:
                             lastPageRank = seqMeta["max_rank"] - limit
-                            if lastPageRank < bounds.max:
-                                lastPageRank = bounds.max
+                            if lastPageRank < bounds.last:
+                                lastPageRank = bounds.last
 
                     links.append(
                         {
@@ -608,19 +605,13 @@ def _getPictureItemById(collectionId, itemId):
                     p.account_id AS account_id,
                     spl.prevpic, spl.prevpicgeojson, spl.nextpic, spl.nextpicgeojson, p.exif,
                     relp.related_pics, p.gps_accuracy_m, p.h_pixel_density,
-                    t.semantics
+                   
+                    get_picture_semantics(p.id) as semantics,
+                    get_picture_annotations(p.id) as annotations
                 FROM pictures p
                 JOIN sequences_pictures sp ON sp.pic_id = p.id
                 JOIN accounts ON p.account_id = accounts.id
                 JOIN sequences s ON sp.seq_id = s.id
-                LEFT JOIN (
-                    SELECT picture_id, json_agg(json_strip_nulls(json_build_object(
-                        'key', key,
-                        'value', value
-                    ))) AS semantics
-                    FROM pictures_semantics
-                    GROUP BY picture_id
-                ) t ON t.picture_id = p.id
                 LEFT JOIN (
                     SELECT
                         p.id,
@@ -735,6 +726,18 @@ def getCollectionItem(collectionId, itemId):
     return stacItem, picStatusToHttpCode[stacItem["properties"]["geovisio:status"]], {"Content-Type": "application/geo+json"}
 
 
+class SearchParams(BaseModel):
+    bbox: Optional[str] = None
+    limit: int = 10
+    datetime: Optional[str] = None
+    place_position: Optional[str] = None
+    place_distance: Optional[str] = None
+    place_fov_tolerance: Optional[int] = None
+    intersects: Optional[str] = None
+    ids: Optional[str] = None
+    collections: Optional[str] = None
+
+
 @bp.route("/search", methods=["GET", "POST"])
 def searchItems():
     """Search through all available items
@@ -755,6 +758,7 @@ def searchItems():
             - $ref: '#/components/parameters/GeoVisio_place_position'
             - $ref: '#/components/parameters/GeoVisio_place_distance'
             - $ref: '#/components/parameters/GeoVisio_place_fov_tolerance'
+            - $ref: '#/components/parameters/searchCQL2_filter'
     post:
         requestBody:
             required: true
@@ -788,14 +792,14 @@ def searchItems():
         args = request.args
 
     # Limit
-    if args.get("limit") is not None:
-        limit = args.get("limit", type=int)
-        if limit is None or limit < 1 or limit > 10000:
-            raise errors.InvalidAPIUsage(_("Parameter limit must be either empty or a number between 1 and 10000"), status_code=400)
-        else:
-            sqlParams["limit"] = limit
-    else:
-        sqlParams["limit"] = 10
+    limit = args.get("limit") or 10
+    try:
+        limit = int(limit)
+        if limit < 1 or limit > 10000:
+            raise ValueError()
+    except ValueError:
+        raise errors.InvalidAPIUsage(_("Parameter limit must be either empty or a number between 1 and 10000"), status_code=400)
+    sqlParams["limit"] = limit
 
     # Bounding box
     bboxarg = parse_bbox(args.getlist("bbox"))
@@ -900,7 +904,7 @@ def searchItems():
             raise errors.InvalidAPIUsage(_("Parameter collections should be a JSON array of strings"), status_code=400)
 
     # To speed up search, if it's a search by id and on only one id, we use the same code as /collections/:cid/items/:id
-    if args.get("ids") is not None and args:
+    if args.get("ids") is not None:
         ids = parse_list(args.get("ids"), paramName="ids")
         if ids and len(ids) == 1:
             picture_id = ids[0]
@@ -917,7 +921,11 @@ def searchItems():
                     200,
                     {"Content-Type": "application/geo+json"},
                 )
-
+    filter_param = args.get("filter")
+    if filter_param is not None:
+        cql_filter = parse_search_filter(filter_param)
+        if cql_filter is not None:
+            sqlWhere.append(cql_filter)
     #
     # Database query
     #
@@ -932,19 +940,12 @@ SELECT * FROM (
         accounts.name AS account_name, 
         p.account_id AS account_id,
         p.exif, p.gps_accuracy_m, p.h_pixel_density,
-        t.semantics
+        get_picture_semantics(p.id) as semantics,
+        get_picture_annotations(p.id) as annotations
     FROM pictures p
     LEFT JOIN sequences_pictures sp ON p.id = sp.pic_id
     LEFT JOIN sequences s ON s.id = sp.seq_id
     LEFT JOIN accounts ON p.account_id = accounts.id
-    LEFT JOIN (
-        SELECT picture_id, json_agg(json_strip_nulls(json_build_object(
-            'key', key,
-            'value', value
-        ))) AS semantics
-        FROM pictures_semantics
-        GROUP BY picture_id
-    ) t ON t.picture_id = p.id
     WHERE {sqlWhere}
     {orderBy}
     LIMIT %(limit)s
@@ -1247,7 +1248,7 @@ def patchCollectionItem(collectionId, itemId, account):
     ---
     tags:
         - Editing
-        - Tags
+        - Semantics
     parameters:
         - name: collectionId
           in: path

geovisio/web/map.py

@@ -352,23 +352,35 @@ def _get_query(z: int, x: int, y: int, onlyForUser: Optional[UUID], additional_f
         sql.SQL("nb_360_pictures"),
         sql.SQL("nb_pictures - nb_360_pictures AS nb_flat_pictures"),
         sql.SQL(
-            """((CASE WHEN nb_pictures = 0 THEN 0 WHEN nb_pictures <= (select PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY nb_pictures) from pictures_grid)
-                THEN nb_pictures::float / (select PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY nb_pictures) from pictures_grid) * 0.5
-                ELSE 0.5 + nb_pictures::float / (SELECT MAX(nb_pictures) FROM pictures_grid) * 0.5
+            """((CASE WHEN nb_pictures = 0 
+                    THEN 0 
+                WHEN nb_pictures <= (SELECT PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY nb_pictures) FILTER (WHERE nb_pictures > 0) FROM pictures_grid)
+                THEN 
+                    nb_pictures::float / (SELECT PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY nb_pictures) FILTER (WHERE nb_pictures > 0) FROM pictures_grid) * 0.5
+                ELSE 
+                    0.5 + nb_pictures::float / (SELECT MAX(nb_pictures) FROM pictures_grid) * 0.5
             END) * 10)::int / 10::float AS coef"""
         ),
         sql.SQL(
-            """((CASE WHEN nb_360_pictures = 0 THEN 0
-                WHEN nb_360_pictures <= (select PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY nb_360_pictures) from pictures_grid)
-                THEN nb_360_pictures::float / (select PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY nb_360_pictures) from pictures_grid) * 0.5
-                ELSE 0.5 + nb_360_pictures::float / (SELECT MAX(nb_360_pictures) FROM pictures_grid) * 0.5
+            """((CASE WHEN nb_360_pictures = 0 
+                    THEN 0
+                WHEN (SELECT PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY nb_360_pictures) FILTER (WHERE nb_360_pictures > 0) FROM pictures_grid) = 0 
+                    THEN 0
+                WHEN nb_360_pictures <= (SELECT PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY nb_360_pictures) FILTER (WHERE nb_360_pictures > 0) FROM pictures_grid)
+                    THEN nb_360_pictures::float / (SELECT PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY nb_360_pictures) FILTER (WHERE nb_360_pictures > 0) FROM pictures_grid) * 0.5
+                ELSE 
+                    0.5 + nb_360_pictures::float / (SELECT MAX(nb_360_pictures) FROM pictures_grid) * 0.5
             END) * 10)::int / 10::float AS coef_360_pictures"""
         ),
         sql.SQL(
-            """((CASE WHEN (nb_pictures - nb_360_pictures) = 0 THEN 0 
-                WHEN (nb_pictures - nb_360_pictures) <= (select PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY (nb_pictures - nb_360_pictures)) from pictures_grid)
-                    THEN (nb_pictures - nb_360_pictures)::float / (select PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY (nb_pictures - nb_360_pictures)) from pictures_grid) * 0.5
-                ELSE 0.5 + (nb_pictures - nb_360_pictures)::float / (SELECT MAX((nb_pictures - nb_360_pictures)) FROM pictures_grid) * 0.5
+            """((CASE WHEN (nb_pictures - nb_360_pictures) = 0 
+                    THEN 0 
+                WHEN (SELECT PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY (nb_pictures - nb_360_pictures)) FILTER (WHERE (nb_pictures - nb_360_pictures) > 0) FROM pictures_grid) = 0 
+                    THEN 0
+                WHEN (nb_pictures - nb_360_pictures) <= (SELECT PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY (nb_pictures - nb_360_pictures)) FILTER (WHERE (nb_pictures - nb_360_pictures) > 0) FROM pictures_grid)
+                    THEN (nb_pictures - nb_360_pictures)::float / (SELECT PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY (nb_pictures - nb_360_pictures)) FILTER (WHERE (nb_pictures - nb_360_pictures) > 0) FROM pictures_grid) * 0.5
+                ELSE 
+                    0.5 + (nb_pictures - nb_360_pictures)::float / (SELECT MAX((nb_pictures - nb_360_pictures)) FROM pictures_grid) * 0.5
             END) * 10)::int / 10::float AS coef_flat_pictures"""
         ),
     ]
@@ -616,7 +628,7 @@ def getUserTile(userId: UUID, z: int, x: int, y: int, format: str):
                         format: binary
     """
 
-    filter = params.parse_filter(request.args.get("filter"))
+    filter = params.parse_collection_filter(request.args.get("filter"))
     return _getTile(z, x, y, format, onlyForUser=userId, filter=filter)
 
 
@@ -688,5 +700,5 @@ def getMyTile(account: Account, z: int, x: int, y: int, format: str):
                         type: string
                         format: binary
     """
-    filter = params.parse_filter(request.args.get("filter"))
+    filter = params.parse_collection_filter(request.args.get("filter"))
     return _getTile(z, x, y, format, onlyForUser=UUID(account.id), filter=filter)

geovisio/web/params.py

@@ -8,8 +8,6 @@ import datetime
 import re
 from werkzeug.datastructures import MultiDict
 from typing import Optional, Tuple, Any, List
-from pygeofilter.backends.sql import to_sql_where
-from pygeofilter.parsers.ecql import parse as ecql_parser
 from pygeofilter import ast
 from pygeofilter.backends.evaluator import Evaluator, handle
 from psycopg import sql
@@ -17,6 +15,8 @@ from geovisio.utils.sequences import STAC_FIELD_MAPPINGS, STAC_FIELD_TO_SQL_FILT
 from geovisio.utils.fields import SortBy, SQLDirection, SortByField
 from flask_babel import gettext as _
 
+from geovisio.utils.cql2 import parse_cql2_filter
+
 
 RGX_SORTBY = re.compile("[+-]?[A-Za-z_].*(,[+-]?[A-Za-z_].*)*")
 SEQUENCES_DEFAULT_FETCH = 100
@@ -325,37 +325,27 @@ def parse_list(value: Optional[Any], tryFallbacks: bool = True, paramName: Optio
         return None
 
 
-def parse_filter(value: Optional[str]) -> Optional[sql.SQL]:
+def parse_collection_filter(value: Optional[str]) -> Optional[sql.SQL]:
     """Reads STAC filter parameter and sends SQL condition back.
 
-    >>> parse_filter('')
+    >>> parse_collection_filter('')
 
-    >>> parse_filter("updated >= '2023-12-31'")
+    >>> parse_collection_filter("updated >= '2023-12-31'")
     SQL("(s.updated_at >= '2023-12-31')")
-    >>> parse_filter("updated >= '2023-12-31' AND created < '2023-10-31'")
+    >>> parse_collection_filter("updated >= '2023-12-31' AND created < '2023-10-31'")
     SQL("((s.updated_at >= '2023-12-31') AND (s.inserted_at < '2023-10-31'))")
-    >>> parse_filter("status IN ('deleted','ready')") # when we ask for deleted, we should also have hidden collections
+    >>> parse_collection_filter("status IN ('deleted','ready')") # when we ask for deleted, we should also have hidden collections
     SQL("s.status IN ('deleted', 'ready', 'hidden')")
-    >>> parse_filter("status = 'deleted' OR status = 'ready'")
+    >>> parse_collection_filter("status = 'deleted' OR status = 'ready'")
     SQL("(((s.status = 'deleted') OR (s.status = 'hidden')) OR (s.status = 'ready'))")
-    >>> parse_filter('invalid = 10') # doctest: +IGNORE_EXCEPTION_DETAIL
+    >>> parse_collection_filter('invalid = 10') # doctest: +IGNORE_EXCEPTION_DETAIL
     Traceback (most recent call last):
     geovisio.errors.InvalidAPIUsage: Unsupported filter parameter
-    >>> parse_filter('updated == 10') # doctest: +IGNORE_EXCEPTION_DETAIL
+    >>> parse_collection_filter('updated == 10') # doctest: +IGNORE_EXCEPTION_DETAIL
     Traceback (most recent call last):
     geovisio.errors.InvalidAPIUsage: Unsupported filter parameter
     """
-    if value is not None and len(value) > 0:
-        try:
-            filterAst = ecql_parser(value)
-            altered_ast = _alterFilterAst(filterAst)  # type: ignore
-
-            f = to_sql_where(altered_ast, STAC_FIELD_TO_SQL_FILTER).replace('"', "")
-            return sql.SQL(f)  # type: ignore
-        except:
-            raise errors.InvalidAPIUsage(_("Unsupported filter parameter"), status_code=400)
-    else:
-        return None
+    return parse_cql2_filter(value, STAC_FIELD_TO_SQL_FILTER, ast_updater=_alterFilterAst)
 
 
 def parse_picture_heading(heading: Optional[str]) -> Optional[int]:

geovisio/web/stac.py

@@ -20,10 +20,11 @@ from geovisio.utils.sequences import (
     get_collections,
     CollectionsRequest,
     STAC_FIELD_MAPPINGS,
+    get_dataset_bounds,
     get_pagination_links,
 )
 from geovisio.web.params import (
-    parse_filter,
+    parse_collection_filter,
     parse_collections_limit,
 )
 
@@ -334,12 +335,17 @@ def getUserCatalog(userId, userIdMatchesAccount=False):
     """
 
     collection_request = CollectionsRequest(
-        sort_by=SortBy(fields=[SortByField(field=STAC_FIELD_MAPPINGS["created"], direction=SQLDirection.ASC)]),
+        sort_by=SortBy(
+            fields=[
+                SortByField(field=STAC_FIELD_MAPPINGS["created"], direction=SQLDirection.ASC),
+                SortByField(field=STAC_FIELD_MAPPINGS["id"], direction=SQLDirection.ASC),
+            ]
+        ),
         user_id=userId,
         userOwnsAllCollections=userIdMatchesAccount,
     )
     collection_request.limit = parse_collections_limit(request.args.get("limit"))
-    collection_request.pagination_filter = parse_filter(request.args.get("page"))
+    collection_request.pagination_filter = parse_collection_filter(request.args.get("page"))
 
     userName = None
     meta_collection = None
@@ -350,12 +356,14 @@ def getUserCatalog(userId, userIdMatchesAccount=False):
             raise errors.InvalidAPIUsage(_("Impossible to find user %(u)s", u=userId))
         userName = userName["name"]
 
-        meta_collection = cursor.execute(
-            SQL("SELECT MIN(inserted_at) AS min_order, MAX(inserted_at) AS max_order FROM sequences s WHERE account_id = %(account)s"),
-            params={"account": userId},
-        ).fetchone()
+        datasetBounds = get_dataset_bounds(
+            cursor.connection,
+            collection_request.sort_by,
+            additional_filters=SQL("s.account_id = %(account)s"),
+            additional_filters_params={"account": userId},
+        )
 
-        if not meta_collection or meta_collection["min_order"] is None:
+        if datasetBounds is None:
             # No data found, trying to give the most meaningfull error message
             raise errors.InvalidAPIUsage(_("No data loaded for user %(u)s", u=userId), 404)
 
@@ -388,10 +396,9 @@ def getUserCatalog(userId, userIdMatchesAccount=False):
     pagination_links = get_pagination_links(
         route="stac.getUserCatalog",
         routeArgs={"userId": str(userId), "limit": collection_request.limit},
-        field=collection_request.sort_by.fields[0].field.stac,
-        direction=collection_request.sort_by.fields[0].direction,
-        datasetBounds=Bounds(min=meta_collection["min_order"], max=meta_collection["max_order"]),
-        dataBounds=db_collections.query_first_order_bounds,
+        sortBy=collection_request.sort_by,
+        datasetBounds=datasetBounds,
+        dataBounds=db_collections.query_bounds,
         additional_filters=None,
     )
 

geovisio/web/upload_set.py

@@ -98,7 +98,7 @@ def create_upload_set(params: UploadSetCreationParameter, accountId: UUID) -> Up
     )
 
     if db_upload_set is None:
-        raise Exception("Impossible to insert sequence in database")
+        raise Exception("Impossible to insert upload_set in database")
 
     return db_upload_set
 
@@ -106,13 +106,6 @@ def create_upload_set(params: UploadSetCreationParameter, accountId: UUID) -> Up
 def update_upload_set(upload_set_id: UUID, params: UploadSetUpdateParameter) -> UploadSet:
     db_params = model_query.get_db_params_and_values(params)
 
-    with db.conn(current_app) as conn, conn.transaction():
-        import psycopg
-
-        cur = psycopg.ClientCursor(conn)
-        q = SQL("UPDATE upload_sets SET {fields} WHERE id = %(upload_set_id)s").format(fields=db_params.fields_for_set())
-        print(cur.mogrify(q, db_params.params_as_dict | {"upload_set_id": upload_set_id}))
-
     with db.execute(
         current_app,
         SQL("UPDATE upload_sets SET {fields} WHERE id = %(upload_set_id)s").format(fields=db_params.fields_for_set()),
@@ -201,7 +194,7 @@ def patchUploadSet(upload_set_id, account=None):
         content:
             application/json:
                 schema:
-                    $ref: '#/components/schemas/GeoVisioUploadSet'
+                    $ref: '#/components/schemas/UploadSetUpdateParameter'
     security:
         - bearerToken: []
         - cookieAuth: []

geovisio/workers/runner_pictures.py

@@ -2,7 +2,7 @@ from fs.path import dirname
 from PIL import Image, ImageOps
 from flask import current_app
 from geovisio import utils
-from geovisio.utils import db, sequences, upload_set
+from geovisio.utils import db, semantics, sequences, upload_set
 import psycopg
 from psycopg.rows import dict_row
 from psycopg.sql import SQL
@@ -22,8 +22,6 @@ import geovisio.utils.filesystems
 
 log = logging.getLogger("geovisio.runner_pictures")
 
-PROCESS_MAX_RETRY = 5  # Number of times a job will be retryed if there is a `RecoverableProcessException` during process (like if the blurring api is not reachable).
-
 
 class PictureBackgroundProcessor(object):
     def __init__(self, app):
@@ -50,6 +48,10 @@ class PictureBackgroundProcessor(object):
             worker = PictureProcessor(app=current_app)
             return self.executor.submit(worker.process_jobs)
 
+    def stop(self):
+        if self.enabled:
+            self.executor.shutdown(cancel_futures=True, wait=True)
+
 
 class ProcessTask(str, Enum):
     prepare = "prepare"
@@ -103,6 +105,55 @@ class DbJob:
         return f"{self.task} for {impacted_object}"
 
 
+def store_detection_semantics(pic: DbPicture, metadata: Dict[str, Any], store_id: bool):
+    """store the detection returned by the blurring API in the database.
+
+    The semantics part is stored as annotations, linked to the default account.
+
+    The blurring id, which could be used to unblur the picture later, is stored in a separate column?
+
+    Note that all old semantics tags are removed, and to know this, we check the `service_name` field returned by the blurring API, and the special qualifier tag
+    `detection_model` that is formated like a user-agent.
+    So we delete all old tags (and related qualifiers) o
+    """
+    from geovisio.utils import annotations
+
+    tags = metadata.pop("annotations", [])
+
+    with db.conn(current_app) as conn, conn.cursor() as cursor:
+        blurring_id = metadata.get("blurring_id")
+        if blurring_id and store_id:
+            # we store the blurring id to be able to unblur the picture later
+            cursor.execute(
+                "UPDATE pictures SET blurring_id = %(blurring_id)s WHERE id = %(id)s",
+                {"blurring_id": blurring_id, "id": pic.id},
+            )
+
+        if not tags:
+            return
+
+        default_account_id = cursor.execute("SELECT id from accounts where is_default = true").fetchone()
+        if not default_account_id:
+            log.error("Impossible to find a default account, cannot add semantics from blurring api")
+        default_account_id = default_account_id[0]
+
+        # we want to remove all the tags added by the same bluring api previously
+        # it's especially usefull when a picture is blurred multiple times
+        # and if the detection model has been updated between the blurrings
+        semantics.delete_annotation_tags_from_service(conn, pic.id, service_name="SGBlur", account=default_account_id)
+    try:
+        annotations_to_create = [
+            annotations.AnnotationCreationParameter(**t, account_id=default_account_id, picture_id=pic.id) for t in tags
+        ]
+    except Exception as e:
+        # if the detections are not in the correct format, we skip them
+        msg = errors.getMessageFromException(e)
+        log.error(f"impossible to save blurring detections, skipping it for picture {pic.id}: {msg}")
+        return
+    for a in annotations_to_create:
+        annotations.creation_annotation(a)
+
+
 def processPictureFiles(pic: DbPicture, config):
     """Generates the files associated with a sequence picture.
 
@@ -140,15 +191,23 @@ def processPictureFiles(pic: DbPicture, config):
         if not skipBlur:
             with sentry_sdk.start_span(description="Blurring picture"):
                 try:
-                    picture = utils.pictures.createBlurredHDPicture(
+                    res = utils.pictures.createBlurredHDPicture(
                         fses.permanent,
                         config.get("API_BLUR_URL"),
                         pictureBytes,
                         picHdPath,
+                        keep_unblured_parts=config["PICTURE_PROCESS_KEEP_UNBLURRED_PARTS"],
                     )
                 except Exception as e:
-                    log.exception(f"impossible to blur picture {pic.id}")
-                    raise RecoverableProcessException("Blur API failure: " + errors.getMessageFromException(e)) from e
+                    msg = errors.getMessageFromException(e)
+                    log.error(f"impossible to blur picture {pic.id}: {msg}")
+                    raise RecoverableProcessException("Blur API failure: " + msg) from e
+                if res is None:
+                    picture = None
+                else:
+                    picture = res.image
+                    if res.metadata:
+                        store_detection_semantics(pic, res.metadata, store_id=config["PICTURE_PROCESS_KEEP_UNBLURRED_PARTS"])
 
                 # Delete original unblurred file
                 geovisio.utils.filesystems.removeFsEvenNotFound(fses.tmp, picHdPath)
@@ -383,12 +442,13 @@ def _get_next_job(app):
                         _finalize_job(locking_transaction, job)
                         log.debug(f"Job {job.label()} processed")
                     except RecoverableProcessException as e:
-                        _mark_process_as_error(locking_transaction, job, e, recoverable=True)
+                        _mark_process_as_error(locking_transaction, job, e, config=app.config, recoverable=True)
                     except RetryLaterProcessException as e:
                         _mark_process_as_error(
                             locking_transaction,
                             job,
                             e,
+                            config=app.config,
                             recoverable=True,
                             mark_as_error=False,
                         )
@@ -396,11 +456,11 @@ def _get_next_job(app):
                         log.error(f"Interruption received, stoping job {job.label()}")
                         # starts a new connection, since the current one can be corrupted by the exception
                         with app.pool.connection() as t:
-                            _mark_process_as_error(t, job, interruption, recoverable=True)
+                            _mark_process_as_error(t, job, interruption, config=app.config, recoverable=True)
                         error = interruption
                     except Exception as e:
                         log.exception(f"Impossible to finish job {job.label()}")
-                        _mark_process_as_error(locking_transaction, job, e, recoverable=False)
+                        _mark_process_as_error(locking_transaction, job, e, config=app.config, recoverable=False)
 
                         # try to finalize the sequence anyway
                         _finalize_sequence(job)
@@ -506,6 +566,7 @@ def _mark_process_as_error(
     conn,
     job: DbJob,
     e: Exception,
+    config: Dict,
     recoverable: bool = False,
     mark_as_error: bool = True,
 ):
@@ -524,7 +585,7 @@ def _mark_process_as_error(
                 RETURNING nb_errors""",
                 {"err": str(e), "id": job.job_queue_id},
             ).fetchone()
-            if nb_error and nb_error[0] > PROCESS_MAX_RETRY:
+            if nb_error and nb_error[0] > config["PICTURE_PROCESS_NB_RETRIES"]:
                 logging.info(f"Job {job.label()} has failed {nb_error} times, we stop trying to process it.")
                 recoverable = False
         else: