geovisio 2.8.0__py3-none-any.whl → 2.9.0__py3-none-any.whl

geovisio/utils/semantics.py

@@ -1,42 +1,20 @@
+from ast import Dict
+from collections import defaultdict
 from dataclasses import dataclass
+import re
 from uuid import UUID
-from psycopg import Cursor
-from psycopg.sql import SQL, Identifier
+from psycopg import Connection, Cursor
+from psycopg.sql import SQL, Identifier, Placeholder
 from psycopg.types.json import Jsonb
 from psycopg.errors import UniqueViolation
-from pydantic import BaseModel, ConfigDict, Field
-from typing import List
+from psycopg.rows import dict_row
+from typing import Generator, List, Optional
 from enum import Enum
 
 from geovisio import errors
-
-
-class TagAction(str, Enum):
-    """Actions to perform on a tag list"""
-
-    add = "add"
-    delete = "delete"
-
-
-class SemanticTagUpdate(BaseModel):
-    """Parameters used to update a tag list"""
-
-    action: TagAction = Field(default=TagAction.add)
-    """Action to perform on the tag list. The default action is `add` which will add the given tag to the list.
-    The action can also be to `delete` the key/value"""
-    key: str = Field(max_length=256)
-    """Key of the tag to update limited to 256 characters"""
-    value: str = Field(max_length=2048)
-    """Value of the tag to update limited ot 2048 characters"""
-
-    model_config = ConfigDict(use_attribute_docstrings=True)
-
-
-class SemanticTag(BaseModel):
-    key: str
-    """Key of the tag"""
-    value: str
-    """Value of the tag"""
+from geovisio.utils.annotations import Annotation, get_picture_annotations
+from geovisio.utils.pic_shape import Geometry
+from geovisio.utils.tags import SemanticTag, SemanticTagUpdate, TagAction
 
 
 class EntityType(Enum):
@@ -45,9 +23,6 @@ class EntityType(Enum):
     seq = "sequence_id"
     annotation = "annotation_id"
 
-    def entitiy_id_field(self) -> Identifier:
-        return Identifier(self.value)
-
 
 @dataclass
 class Entity:
@@ -72,49 +47,184 @@ class Entity:
             case EntityType.seq:
                 return Identifier("sequences_semantics_history")
             case EntityType.annotation:
-                return Identifier("annotations_semantics_history")
+                return Identifier("pictures_semantics_history")
             case _:
                 raise ValueError(f"Unknown entity type: {self.type}")
 
 
-def update_tags(cursor: Cursor, entity: Entity, actions: List[SemanticTagUpdate], account: UUID) -> SemanticTag:
+def update_tags(cursor: Cursor, entity: Entity, actions: List[SemanticTagUpdate], account: UUID, annotation=None):
     """Update tags for an entity
     Note: this should be done inside an autocommit transaction
     """
     table_name = entity.get_table()
-    fields = [entity.type.entitiy_id_field(), Identifier("key"), Identifier("value")]
     tag_to_add = [t for t in actions if t.action == TagAction.add]
     tag_to_delete = [t for t in actions if t.action == TagAction.delete]
     try:
         if tag_to_delete:
-            cursor.execute(SQL("CREATE TEMPORARY TABLE tags_to_delete(key TEXT, value TEXT) ON COMMIT DROP"))
-            with cursor.copy(SQL("COPY tags_to_delete (key, value) FROM STDIN")) as copy:
-                for tag in tag_to_delete:
-                    copy.write_row((tag.key, tag.value))
+            filter_query = []
+            params = [entity.id]
+            for tag in tag_to_delete:
+                filter_query.append(SQL("(key = %s AND value = %s)"))
+                params.append(tag.key)
+                params.append(tag.value)
+
             cursor.execute(
                 SQL(
                     """DELETE FROM {table}
-    WHERE {entity_id} = %(entity)s 
-    AND (key, value) IN (
-        SELECT key, value FROM tags_to_delete
-    )"""
-                ).format(table=table_name, entity_id=entity.type.entitiy_id_field()),
-                {"entity": entity.id, "key_values": [(t.key, t.value) for t in tag_to_delete]},
+WHERE {entity_id} = %s 
+AND ({filter})"""
+                ).format(table=table_name, entity_id=Identifier(entity.type.value), filter=SQL(" OR ").join(filter_query)),
+                params,
             )
         if tag_to_add:
-            with cursor.copy(SQL("COPY {table} ({fields}) FROM STDIN").format(table=table_name, fields=SQL(",").join(fields))) as copy:
+            with cursor.copy(
+                SQL("COPY {table} ({fields}) FROM STDIN").format(
+                    table=table_name,
+                    fields=SQL(",").join([Identifier(entity.type.value), Identifier("key"), Identifier("value")]),
+                )
+            ) as copy:
                 for tag in tag_to_add:
                     copy.write_row((entity.id, tag.key, tag.value))
         if tag_to_add or tag_to_delete:
             # we track the history changes of the semantic tags
-            cursor.execute(
-                SQL("INSERT INTO {history_table} ({entity_id_field}, account_id, updates) VALUES (%(id)s, %(account)s, %(tags)s)").format(
-                    history_table=entity.get_history_table(), entity_id_field=entity.type.entitiy_id_field()
-                ),
-                {"id": entity.id, "account": account, "tags": Jsonb([t.model_dump() for t in tag_to_add + tag_to_delete])},
-            )
+            track_semantic_history(cursor, entity, actions, account, annotation)
     except UniqueViolation as e:
         # if the tag already exists, we don't want to add it again
         raise errors.InvalidAPIUsage(
             "Impossible to add semantic tags because of duplicates", payload={"details": {"duplicate": e.diag.message_detail}}
         )
+
+
+class SemanticTagUpdateOnAnnotation(SemanticTagUpdate):
+    annotation_shape: Geometry
+
+
+def track_semantic_history(cursor: Cursor, entity: Entity, actions: List[SemanticTagUpdate], account: UUID, annotation):
+    params = {
+        "account_id": account,
+    }
+    if annotation is not None:
+        # the annotations are historized in the pictures_semantics_history table
+        # and additional information about the annotation.
+        # This makes it easier to track annotations deletions
+        params["picture_id"] = annotation.picture_id
+
+        params["updates"] = Jsonb(
+            [
+                SemanticTagUpdateOnAnnotation(action=t.action, key=t.key, value=t.value, annotation_shape=annotation.shape).model_dump()
+                for t in actions
+            ]
+        )
+    else:
+        params[entity.type.value] = entity.id
+        params["updates"] = Jsonb([t.model_dump() for t in actions])
+
+    sql = SQL("INSERT INTO {history_table} ({fields}) VALUES ({values})").format(
+        history_table=entity.get_history_table(),
+        fields=SQL(", ").join([Identifier(k) for k in params.keys()]),
+        values=SQL(", ").join([Placeholder(k) for k in params.keys()]),
+    )
+    cursor.execute(sql, params)
+
+
+def delete_annotation_tags_from_service(conn: Connection, picture_id: UUID, service_name: str, account: UUID) -> List[Dict]:
+    """Delete all tags from a blurring service on a given picture"""
+    annotations_tags = list(get_annotation_tags_from_service(conn, picture_id, service_name))
+
+    with conn.transaction(), conn.cursor(row_factory=dict_row) as cursor:
+        for a in annotations_tags:
+            actions = [SemanticTagUpdate(action=TagAction.delete, key=t.key, value=t.value) for t in a.semantics]
+            entity = Entity(id=a.id, type=EntityType.annotation)
+            update_tags(cursor, entity, actions, account, annotation=a)
+
+    return annotations_tags
+
+
+QUALIFIER_REGEXP = re.compile(r"^(?P<qualifier>[^\[]*)\[(?P<key>[^=]+)(=(?P<value>.*))?\]$")
+
+
+@dataclass
+class QualifierSemantic:
+    qualifier: str
+    associated_key: str
+    associated_value: Optional[str]
+    raw_tag: SemanticTag
+
+    def qualifies(self, semantic_tag: SemanticTag) -> bool:
+        """Check if a semantic tag is qualified by the qualifier"""
+        if semantic_tag.key != self.associated_key:
+            return False
+        if self.associated_value is None or self.associated_value == "*":
+            return True
+        return semantic_tag.value == self.associated_value
+
+
+def as_qualifier(s: SemanticTag) -> Optional[QualifierSemantic]:
+    """Try to convert a semantic tag into a qualifier"""
+    m = QUALIFIER_REGEXP.search(s.key)
+    if m:
+        return QualifierSemantic(
+            qualifier=m.group("qualifier"),
+            associated_key=m.group("key"),
+            associated_value=m.group("value"),
+            raw_tag=s,
+        )
+
+
+def get_qualifiers(semantics: List[SemanticTag]) -> List[QualifierSemantic]:
+    """Find all qualifiers in a list of semantic tags"""
+    res = []
+    for s in semantics:
+        q = as_qualifier(s)
+        if q is not None:
+            res.append(q)
+    return res
+
+
+def find_detection_model_tags(qualifiers: List[QualifierSemantic], service_name: str) -> List[QualifierSemantic]:
+    """Find all detection models associated to a picture, from a given service"""
+    res = []
+    for q in qualifiers:
+        if not q.raw_tag.value.startswith(f"{service_name}-"):
+            continue
+        if q.qualifier != "detection_model":
+            continue
+        res.append(q)
+    return res
+
+
+def find_semantics_from_service(annotation, service_name: str) -> Generator[SemanticTag, None, None]:
+    """Find all semantics tags related from a given bluring service
+
+    The blurring service will add a `detection_model` qualifier with a value starting by its name (like `SGBlur-yolo11n/0.1.0` for `SGBlur`)
+
+    This method will return all linked semantics tags, and all their qualifiers.
+    """
+    qualifiers = get_qualifiers(annotation.semantics)
+    detection_model_tags = find_detection_model_tags(qualifiers, service_name)
+    qualified_tags = []
+    for s in annotation.semantics:
+        for qualifier_tag in detection_model_tags:
+            if qualifier_tag.qualifies(s):
+                qualified_tags.append(s)
+                break
+
+    # we then have to get all qualifiers on those tags
+    related_qualifiers = []
+    for q in qualifiers:
+        for t in qualified_tags:
+            if q.qualifies(t):
+                related_qualifiers.append(q.raw_tag)
+                break
+
+    return qualified_tags + related_qualifiers
+
+
+def get_annotation_tags_from_service(conn: Connection, picture_id: UUID, service_name: str) -> Generator[Annotation, None, None]:
+    """Get all annotations semantics from a blurring service"""
+
+    annotations = get_picture_annotations(conn, picture_id)
+
+    for a in annotations:
+        semantics = [s for s in find_semantics_from_service(a, service_name)]
+        yield Annotation(id=a.id, picture_id=a.picture_id, shape=a.shape, semantics=semantics)

geovisio/utils/sentry.py

@@ -60,7 +60,6 @@ def _wrap_cursor_execute(f):
             return f(self, query, params, prepare=prepare, binary=binary)
 
         with record_sql_queries(
-            hub=hub,
             cursor=self,
             query=query,
             params_list=params,

geovisio/utils/sequences.py

@@ -1,5 +1,8 @@
+from operator import ne
+from click import Option
+from numpy import sort
 import psycopg
-from flask import current_app, url_for
+from flask import current_app, g, url_for
 from flask_babel import gettext as _
 from psycopg.types.json import Jsonb
 from psycopg.sql import SQL, Composable
@@ -11,7 +14,7 @@ from uuid import UUID
 from enum import Enum
 from geovisio.utils import db
 from geovisio.utils.auth import Account
-from geovisio.utils.fields import FieldMapping, SortBy, SQLDirection, BBox, Bounds
+from geovisio.utils.fields import FieldMapping, SortBy, SQLDirection, BBox, Bounds, SortByField
 from geopic_tag_reader import reader
 from pathlib import PurePath
 from geovisio import errors, utils
@@ -39,6 +42,7 @@ STAC_FIELD_MAPPINGS = {
         FieldMapping(sql_column=SQL("updated_at"), stac="updated"),
         FieldMapping(sql_column=SQL("computed_capture_date"), stac="datetime"),
         FieldMapping(sql_column=SQL("status"), stac="status"),
+        FieldMapping(sql_column=SQL("id"), stac="id"),
     ]
 }
 STAC_FIELD_TO_SQL_FILTER = {p.stac: p.sql_filter.as_string(None) for p in STAC_FIELD_MAPPINGS.values()}
@@ -52,7 +56,7 @@ class Collections:
 
     collections: List[Dict[Any, Any]] = field(default_factory=lambda: [])
     # Bounds of the field used by the first field of the `ORDER BY` (usefull especially for pagination)
-    query_first_order_bounds: Optional[Bounds] = None
+    query_bounds: Optional[Bounds] = None
 
 
 @dataclass
@@ -166,7 +170,7 @@ def get_collections(request: CollectionsRequest) -> Collections:
                     SELECT sequence_id, json_agg(json_strip_nulls(json_build_object(
                         'key', key,
                         'value', value
-                    ))) AS semantics
+                    )) ORDER BY key, value) AS semantics
                     FROM sequences_semantics
                     GROUP BY sequence_id
                 ) t ON t.sequence_id = s.id
@@ -184,71 +188,150 @@ def get_collections(request: CollectionsRequest) -> Collections:
 
         # Different request if we want the last n sequences
         #  Useful for paginating from last page to first
-        if request.pagination_filter and (
-            (
-                request.sort_by.fields[0].direction == SQLDirection.ASC
-                and request.pagination_filter.as_string(None).startswith(f"({request.sort_by.fields[0].field.sql_filter.as_string(None)} <")
-            )
-            or (
-                request.sort_by.fields[0].direction == SQLDirection.DESC
-                and request.pagination_filter.as_string(None).startswith(f"({request.sort_by.fields[0].field.sql_filter.as_string(None)} >")
-            )
-        ):
-            base_query = sqlSequencesRaw.format(
-                filter=SQL(" AND ").join(seq_filter),
-                order1=request.sort_by.revert(),
-                limit=request.limit,
-                status=status_field,
-            )
-            sqlSequences = SQL(
+        if request.pagination_filter:
+            # note: we don't want to compare the leading parenthesis
+            pagination = request.pagination_filter.as_string(None).strip("(")
+            first_sort = request.sort_by.fields[0]
+            if (first_sort.direction == SQLDirection.ASC and pagination.startswith(f"{first_sort.field.sql_filter.as_string(None)} <")) or (
+                first_sort.direction == SQLDirection.DESC and pagination.startswith(f"{first_sort.field.sql_filter.as_string(None)} >")
+            ):
+                base_query = sqlSequencesRaw.format(
+                    filter=SQL(" AND ").join(seq_filter),
+                    order1=request.sort_by.revert(),
+                    limit=request.limit,
+                    status=status_field,
+                )
+                sqlSequences = SQL(
+                    """
+                    SELECT *
+                    FROM ({base_query}) s
+                    ORDER BY {order2}
                 """
-                SELECT *
-                FROM ({base_query}) s
-                ORDER BY {order2}
-            """
-            ).format(
-                order2=request.sort_by.as_sql(),
-                base_query=base_query,
-            )
+                ).format(
+                    order2=request.sort_by.as_sql(),
+                    base_query=base_query,
+                )
 
         records = cursor.execute(sqlSequences, seq_params).fetchall()
 
         query_bounds = None
-        for s in records:
-            first_order_val = s.get(request.sort_by.fields[0].field.stac)
-            if first_order_val is None:
-                continue
-            if query_bounds is None:
-                query_bounds = Bounds(first_order_val, first_order_val)
-            else:
-                query_bounds.update(first_order_val)
+        if records:
+            first = [records[0].get(f.field.stac) for f in request.sort_by.fields]
+            last = [records[-1].get(f.field.stac) for f in request.sort_by.fields]
+            query_bounds = Bounds(first, last)
 
         return Collections(
             collections=records,
-            query_first_order_bounds=query_bounds,
+            query_bounds=query_bounds,
         )
 
 
+def get_pagination_stac_filter(sortBy: SortBy, dataBounds: Optional[Bounds[List[Any]]], next: bool) -> str:
+    """Create a pagination API filters, using the sorts and the bounds of the current query"""
+    filters = []
+    bounds = dataBounds.last if next else dataBounds.first
+    for i, f in enumerate(sortBy.fields):
+        direction = f.direction
+        # bounds is a list of values, for all sorty_by fields
+        if (next and direction == SQLDirection.ASC) or (not next and direction == SQLDirection.DESC):
+            cmp = ">"
+        else:
+            cmp = "<"
+        field_pagination = f"{f.field.stac} {cmp} '{bounds[i]}'"
+
+        previous_filters = sortBy.fields[:i]
+        if previous_filters:
+            prev_fields = " AND ".join([f"{f.field.stac} = '{bounds[prev_i]}'" for prev_i, f in enumerate(previous_filters)]) + " AND "
+            filters.append(f"({prev_fields}{field_pagination})")
+        else:
+            filters.append(field_pagination)
+    return " OR ".join(filters)
+
+
+def get_dataset_bounds(
+    conn: psycopg.Connection,
+    sortBy: SortBy,
+    additional_filters: Optional[SQL] = None,
+    additional_filters_params: Optional[Dict[str, Any]] = None,
+) -> Optional[Bounds]:
+    """Computes the dataset bounds from the sortBy field (using lexicographic order)
+
+    if there are several sort-by fields like (inserted_at, updated_at), this will return a bound with minimum (resp maximum)
+    inserted_at value, and for this value, the minimum (resp maximum) updated_at value.
+    """
+    with conn.cursor() as cursor:
+
+        sql_bounds = cursor.execute(
+            SQL(
+                """WITH min_bounds AS (
+    SELECT {fields} from sequences s WHERE {filters} ORDER BY {ordered_fields} LIMIT 1
+),
+max_bounds AS (
+    SELECT {fields} from sequences s WHERE {filters} ORDER BY {reverse_fields} LIMIT 1
+)
+SELECT * FROM min_bounds, max_bounds;
+            """
+            ).format(
+                fields=SQL(", ").join([f.field.sql_column for f in sortBy.fields]),
+                ordered_fields=sortBy.as_non_aliased_sql(),
+                reverse_fields=sortBy.revert_non_aliased_sql(),
+                filters=additional_filters or SQL("TRUE"),
+            ),
+            params=additional_filters_params or {},
+        ).fetchone()
+        if not sql_bounds:
+            return None
+        min = [sql_bounds[i] for i, f in enumerate(sortBy.fields)]
+        max = [sql_bounds[i + len(sortBy.fields)] for i, f in enumerate(sortBy.fields)]
+        return Bounds(first=min, last=max)
+
+
+def has_previous_results(sortBy: SortBy, datasetBounds: Bounds, dataBounds: Bounds) -> bool:
+    """Check if there are results in the database before the one returned by the queries
+    To do this, we do a lexicographic comparison of the bounds, using the fields direction
+
+    Note: the bounds are reversed for the DESC direction, so the bounds.min >= bounds.max for the DESC direction
+    """
+    for i, f in enumerate(sortBy.fields):
+        if dataBounds.first[i] is None or datasetBounds.first[i] is None:
+            continue
+        if (f.direction == SQLDirection.ASC and dataBounds.first[i] > datasetBounds.first[i]) or (
+            f.direction == SQLDirection.DESC and datasetBounds.first[i] > dataBounds.first[i]
+        ):
+            return True
+    return False
+
+
+def has_next_results(sortBy: SortBy, datasetBounds: Bounds, dataBounds: Bounds) -> bool:
+    """Check if there are results in the database after the one returned by the queries
+    To do this, we do a lexicographic comparison of the bounds, using the fields direction"""
+    for i, f in enumerate(sortBy.fields):
+        if dataBounds.last[i] is None or datasetBounds.last[i] is None:
+            continue
+        if (f.direction == SQLDirection.ASC and dataBounds.last[i] < datasetBounds.last[i]) or (
+            f.direction == SQLDirection.DESC and datasetBounds.last[i] < dataBounds.last[i]
+        ):
+            return True
+    return False
+
+
 def get_pagination_links(
     route: str,
     routeArgs: dict,
-    field: str,
-    direction: SQLDirection,
+    sortBy: SortBy,
     datasetBounds: Bounds,
     dataBounds: Optional[Bounds],
     additional_filters: Optional[str],
 ) -> List:
     """Computes STAC links to handle pagination"""
 
-    sortby = f"{'+' if direction == SQLDirection.ASC else '-'}{field}"
+    sortby = sortBy.as_stac()
     links = []
-    if dataBounds is None:
+    if dataBounds is None or datasetBounds is None:
         return links
 
     # Check if first/prev links are necessary
-    if (direction == SQLDirection.ASC and datasetBounds.min < dataBounds.min) or (
-        direction == SQLDirection.DESC and dataBounds.max < datasetBounds.max
-    ):
+    if has_previous_results(sortBy, datasetBounds=datasetBounds, dataBounds=dataBounds):
         links.append(
             {
                 "rel": "first",
@@ -256,7 +339,9 @@ def get_pagination_links(
                 "href": url_for(route, _external=True, **routeArgs, filter=additional_filters, sortby=sortby),
             }
         )
-        page_filter = f"{field} {'<' if direction == SQLDirection.ASC else '>'} '{dataBounds.min if direction == SQLDirection.ASC else dataBounds.max}'"
+
+        page_filter = get_pagination_stac_filter(sortBy, dataBounds, next=False)
+
         links.append(
             {
                 "rel": "prev",
@@ -273,11 +358,8 @@ def get_pagination_links(
         )
 
     # Check if next/last links are required
-    if (direction == SQLDirection.ASC and dataBounds.max < datasetBounds.max) or (
-        direction == SQLDirection.DESC and datasetBounds.min < dataBounds.min
-    ):
-        next_filter = f"{field} {'>' if direction == SQLDirection.ASC else '<'} '{dataBounds.max if direction == SQLDirection.ASC else dataBounds.min}'"
-        last_filter = f"{field} {'<=' if direction == SQLDirection.ASC else '>='} '{datasetBounds.max if direction == SQLDirection.ASC else datasetBounds.min}'"
+    if has_next_results(sortBy, datasetBounds=datasetBounds, dataBounds=dataBounds):
+        next_filter = get_pagination_stac_filter(sortBy, dataBounds, next=True)
         links.append(
             {
                 "rel": "next",
@@ -292,6 +374,10 @@ def get_pagination_links(
                 ),
             }
         )
+        # for last, we only consider the first field used for sorting, the rest are useless
+        # Note: we compare to the datasetBounds last since it depends on the sort direction (so, for DESC, it last<first)
+        f = sortBy.fields[0]
+        last_filter = f"{f.field.stac} {'<=' if f.direction == SQLDirection.ASC else '>='} '{datasetBounds.last[0]}'"
         links.append(
             {
                 "rel": "last",
@@ -423,15 +509,10 @@ def sort_collection(db, collectionId: UUID, sortby: CollectionSort):
     picForDb = [(collectionId, i + 1, p["id"]) for i, p in enumerate(picMetas)]
 
     # Inject back pictures in sequence
-    db.executemany(
-        SQL(
-            """
-            INSERT INTO sequences_pictures(seq_id, rank, pic_id)
-            VALUES (%s, %s, %s)
-            """
-        ),
-        picForDb,
-    )
+    db.executemany(SQL("INSERT INTO sequences_pictures(seq_id, rank, pic_id) VALUES (%s, %s, %s)"), picForDb)
+
+    # we update the geometry of the sequence after this (the other computed fields have no need for an update)
+    db.execute(SQL("UPDATE sequences SET geom = compute_sequence_geom(id) WHERE id = %s"), [collectionId])
 
 
 def update_headings(

geovisio/utils/tags.py

@@ -0,0 +1,31 @@
+from enum import Enum
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class TagAction(str, Enum):
+    """Actions to perform on a tag list"""
+
+    add = "add"
+    delete = "delete"
+
+
+class SemanticTagUpdate(BaseModel):
+    """Parameters used to update a tag list"""
+
+    action: TagAction = Field(default=TagAction.add)
+    """Action to perform on the tag list. The default action is `add` which will add the given tag to the list.
+    The action can also be to `delete` the key/value"""
+    key: str = Field(max_length=256)
+    """Key of the tag to update limited to 256 characters"""
+    value: str = Field(max_length=2048)
+    """Value of the tag to update limited ot 2048 characters"""
+
+    model_config = ConfigDict(use_attribute_docstrings=True)
+
+
+class SemanticTag(BaseModel):
+    key: str
+    """Key of the tag"""
+    value: str
+    """Value of the tag"""