geovisio 2.8.0__py3-none-any.whl → 2.9.0__py3-none-any.whl

geovisio/utils/annotations.py

@@ -0,0 +1,186 @@
+from typing import List, Optional
+from uuid import UUID
+from flask import current_app
+import psycopg
+from pydantic import BaseModel, Field, field_validator
+from psycopg.sql import SQL
+from psycopg.rows import class_row, dict_row
+
+from geovisio import errors
+from geovisio.utils import db, model_query
+from geovisio.utils import semantics
+from geovisio.utils.pic_shape import InputAnnotationShape, Geometry, get_coords_from_shape, shape_as_geometry
+from geovisio.utils.tags import SemanticTag, SemanticTagUpdate
+
+
+class Annotation(BaseModel):
+    id: UUID
+    """ID of the annotation"""
+    picture_id: UUID
+    """ID of the picture to which the annotation is linked to"""
+    shape: Geometry
+    """Polygon defining the annotation"""
+    semantics: List[SemanticTag] = Field(default_factory=list)
+    """Semantic tags associated to the annotation"""
+
+    @field_validator("semantics", mode="before")
+    @classmethod
+    def parse_semantics(cls, value):
+        return value or []
+
+    @field_validator("shape", mode="before")
+    def validate_shape(cls, value):
+        # if its a bounding box, transform it to a polygon
+        return shape_as_geometry(value)
+
+
+class AnnotationCreationRow(BaseModel):
+    picture_id: UUID
+    """ID of the picture to which the annotation is linked to"""
+    shape: Geometry
+    """shape defining the annotation"""
+
+
+class AnnotationCreationParameter(BaseModel):
+    account_id: UUID
+    """ID of the account that created the annotation"""
+    picture_id: UUID
+    """ID of the picture to which the annotation is linked to"""
+    shape: InputAnnotationShape
+    """Shape defining the annotation.
+The annotation shape is either a full geojson geometry or only a bounding box (4 floats).
+
+The coordinates should be given in pixel, starting from the bottom left of the picture.
+
+Note that the API will always output geometry as geojson geometry (thus will transform the bbox into a polygon).
+"""
+
+    semantics: List[SemanticTagUpdate] = Field(default_factory=list)
+    """Semantic tags associated to the annotation"""
+
+    def shape_as_geometry(self) -> Geometry:
+        return shape_as_geometry(self.shape)
+
+
+def creation_annotation(params: AnnotationCreationParameter) -> Annotation:
+    """Create an annotation in the database"""
+
+    model = model_query.get_db_params_and_values(
+        AnnotationCreationRow(picture_id=params.picture_id, shape=params.shape_as_geometry()), jsonb_fields={"shape"}
+    )
+    insert_query = SQL(
+        """WITH existing_annotations AS (
+        SELECT * FROM annotations WHERE picture_id = %(picture_id)s AND shape = %(shape)s
+)
+, new_ones AS (
+        INSERT INTO annotations (picture_id, shape)
+        SELECT %(picture_id)s, %(shape)s
+        WHERE NOT EXISTS (SELECT FROM existing_annotations)
+        RETURNING *
+)
+SELECT * FROM existing_annotations UNION ALL SELECT * FROM new_ones 
+;"""
+    )
+
+    with db.conn(current_app) as conn, conn.transaction(), conn.cursor(row_factory=class_row(Annotation)) as cursor:
+        # we check that the shape is valid
+        check_shape(conn, params)
+
+        annotation = cursor.execute(insert_query, model.params_as_dict).fetchone()
+
+        if annotation is None:
+            raise Exception("Impossible to insert annotation in database")
+
+        semantics.update_tags(
+            cursor=cursor,
+            entity=semantics.Entity(semantics.EntityType.annotation, annotation.id),
+            actions=params.semantics,
+            account=params.account_id,
+            annotation=annotation,
+        )
+
+        return get_annotation(conn, annotation.id)
+
+
+def check_shape(conn: psycopg.Connection, params: AnnotationCreationParameter):
+    """Check that the shape is valid"""
+    with conn.cursor(row_factory=dict_row) as cursor:
+        picture_size = cursor.execute(
+            SQL("SELECT (metadata->>'width')::int AS width, (metadata->>'height')::int AS height FROM pictures WHERE id = %(pic)s"),
+            {"pic": params.picture_id},
+        ).fetchone()
+
+        for x, y in get_coords_from_shape(params.shape):
+            if x < 0 or x > picture_size["width"] or y < 0 or y > picture_size["height"]:
+                raise errors.InvalidAPIUsage(
+                    message="Annotation shape is outside the range of the picture",
+                    payload={
+                        "details": f"Annotation shape's coordinates should be in pixel, between [0, 0] and [{picture_size['width']}, {picture_size['height']}]",
+                        "value": {"x": x, "y": y},
+                    },
+                )
+
+
+def get_annotation(conn: psycopg.Connection, id: UUID) -> Optional[Annotation]:
+    """Get an annotation in the database"""
+    with conn.cursor(row_factory=class_row(Annotation)) as cursor:
+        return cursor.execute(
+            SQL(
+                """SELECT 
+    id, 
+    shape,
+    picture_id,
+    t.semantics
+    FROM annotations a
+    LEFT JOIN (
+        SELECT annotation_id, json_agg(json_strip_nulls(json_build_object(
+            'key', key,
+            'value', value
+        )) ORDER BY key, value) AS semantics
+        FROM annotations_semantics
+        GROUP BY annotation_id
+    ) t ON t.annotation_id = a.id
+    WHERE a.id = %(id)s"""
+            ),
+            {"id": id},
+        ).fetchone()
+
+
+def get_picture_annotations(conn: psycopg.Connection, picture_id: UUID) -> List[Annotation]:
+    """Get all annotations linked to a picture"""
+    with conn.cursor() as cursor:
+        json_annotations = cursor.execute(
+            SQL("SELECT get_picture_annotations(p.id) FROM pictures p WHERE p.id = %(pic)s"),
+            {"pic": picture_id},
+        ).fetchone()
+        if not json_annotations or not json_annotations[0]:
+            return []
+        return [Annotation(**a, picture_id=picture_id) for a in json_annotations[0]]
+
+
+def update_annotation(annotation: Annotation, tag_updates: List[SemanticTagUpdate], account_id: UUID) -> Optional[Annotation]:
+    """update an annotation in the database.
+    If the annotation has no semantic tags anymore after the update, it will be deleted
+    """
+    with db.conn(current_app) as conn, conn.transaction(), conn.cursor(row_factory=class_row(Annotation)) as cursor:
+
+        semantics.update_tags(
+            cursor=cursor,
+            entity=semantics.Entity(semantics.EntityType.annotation, annotation.id),
+            actions=tag_updates,
+            account=account_id,
+            annotation=annotation,
+        )
+
+        a = get_annotation(conn, annotation.id)
+
+        if a and len(a.semantics) == 0:
+            # the annotation will be deleted by a trigger if its empty
+            return None
+        return a
+
+
+def delete_annotation(conn: psycopg.Connection, annotation_id: UUID) -> None:
+    """Delete an annotation from the database"""
+    with conn.cursor() as cursor:
+        cursor.execute("DELETE FROM annotations WHERE id = %(id)s", {"id": annotation_id})

geovisio/utils/cql2.py

@@ -0,0 +1,134 @@
+from operator import not_
+from lark import UnexpectedCharacters
+from geovisio import errors
+import logging
+from flask_babel import gettext as _
+from psycopg import sql
+from pygeofilter import ast
+from pygeofilter.backends.sql import to_sql_where
+from pygeofilter.backends.evaluator import Evaluator, handle
+from pygeofilter.parsers.ecql import parse as ecql_parser
+from typing import Dict, Optional
+
+
+def parse_cql2_filter(value: Optional[str], field_mapping: Dict[str, str], *, ast_updater=None) -> Optional[sql.SQL]:
+    """Reads CQL2 filter parameter and sends SQL condition back.
+
+    Need a mapping from API field name to database field name.
+
+    And if needed, can take a function to update the AST before sending it to SQL.
+    """
+    if not value:
+        return None
+    try:
+        filterAst = ecql_parser(value)
+
+        if ast_updater:
+            filterAst = ast_updater(filterAst)
+
+        f = to_sql_where(filterAst, field_mapping).replace('"', "").replace("'me'", "%(account_id)s")
+        return sql.SQL(f)  # type: ignore
+    except:
+        logging.error(f"Unsupported filter parameter: {value}")
+        raise errors.InvalidAPIUsage(_("Unsupported filter parameter"), status_code=400)
+
+
+SEMANTIC_FIELD_MAPPOING = {"key": "key", "value": "value"}
+
+
+def parse_semantic_filter(value: Optional[str]) -> Optional[sql.SQL]:
+    """Transform the semantic only filters to SQL
+
+    >>> parse_semantic_filter("semantics.pouet='stop'")
+    SQL("((key = 'pouet') AND (value = 'stop'))")
+    >>> parse_semantic_filter("\\"semantics.osm|traffic_sign\\"='stop'")
+    SQL("((key = 'osm|traffic_sign') AND (value = 'stop'))")
+    """
+    return parse_cql2_filter(value, SEMANTIC_FIELD_MAPPOING, ast_updater=lambda a: SemanticAttributesAstUpdater().evaluate(a))
+
+
+def parse_search_filter(value: Optional[str]) -> Optional[sql.SQL]:
+    """Transform STAC CQL2 filter on /search endpoint to SQL
+
+    Note that, for the moment, only semantics are supported. If more needs to be supported, we should evaluate the
+    non semantic filters separately (likely with a AstEvaluator).
+    """
+    s = parse_semantic_filter(value)
+
+    if s is None:
+        return None
+
+    return sql.SQL(
+        """(p.id in (
+    SELECT picture_id
+    FROM pictures_semantics
+    WHERE {semantic_filter}
+UNION
+    SELECT DISTINCT(picture_id)
+    FROM annotations_semantics ans
+    JOIN annotations a on a.id = ans.annotation_id
+    WHERE {semantic_filter}
+UNION
+    SELECT sp.pic_id
+    FROM sequences_pictures sp
+    join sequences_semantics sm on sp.seq_id = sm.sequence_id
+    WHERE {semantic_filter}
+LIMIT %(limit)s
+))"""
+    ).format(semantic_filter=s)
+
+
+def get_semantic_attribute(node):
+    if isinstance(node, ast.Attribute) and node.name.startswith("semantics."):
+        return node.name[10:]
+    return None
+
+
+class SemanticAttributesAstUpdater(Evaluator):
+    """
+    We alter the AST to handle semantic attributes.
+
+    So
+    * `semantics.some_tag='some_value'` becomes `(key = 'some_tag' AND value = 'some_value')`
+    * `semantics.some_tag IN ('some_value', 'some_other_value')` becomes `(key = 'some_tag' AND value IN ('some_value', 'some_other_value'))`
+    """
+
+    @handle(ast.Equal)
+    def eq(self, node, lhs, rhs):
+        semantic_attribute = get_semantic_attribute(lhs)
+        if semantic_attribute is None:
+            return node
+        return ast.And(ast.Equal(ast.Attribute("key"), semantic_attribute), ast.Equal(ast.Attribute("value"), rhs))
+
+    @handle(ast.Or)
+    def or_(self, node, lhs, rhs):
+        return ast.Or(lhs, rhs)
+
+    @handle(ast.And)
+    def and_(self, node, lhs, rhs):
+        # uncomment this when we know how to handle `AND` in semantic filters
+        # return ast.And(lhs, rhs)
+        raise errors.InvalidAPIUsage("Unsupported filter parameter: AND semantic filters are not yet supported", status_code=400)
+
+    @handle(ast.IsNull)
+    def is_null(self, node, lhs):
+        if not node.not_:
+            raise errors.InvalidAPIUsage(
+                "Unsupported filter parameter: only `IS NOT NULL` is supported (to express that we want all values of a semantic tags)",
+                status_code=400,
+            )
+        semantic_attribute = get_semantic_attribute(lhs)
+        if semantic_attribute is None:
+            return node
+        return ast.Equal(ast.Attribute("key"), semantic_attribute)
+
+    @handle(ast.In)
+    def in_(self, node, lhs, *options):
+        semantic_attribute = get_semantic_attribute(lhs)
+        if semantic_attribute is None:
+            return node
+
+        return ast.And(ast.Equal(ast.Attribute("key"), semantic_attribute), ast.In(ast.Attribute("value"), node.sub_nodes, not_=False))
+
+    def adopt(self, node, *sub_args):
+        return node

geovisio/utils/db.py

@@ -1,4 +1,6 @@
 from psycopg_pool import ConnectionPool
+import psycopg
+from psycopg.abc import Query
 from contextlib import contextmanager
 from typing import Optional
 
@@ -63,3 +65,8 @@ def long_queries_conn(app, connection_timeout: Optional[float] = None):
     """Get a psycopg connection for queries that are known to be long from the connection pool"""
     with app.long_queries_pool.connection(timeout=connection_timeout) as conn:
         yield conn
+
+
+def query_as_string(conn: psycopg.Connection, query: Query, params: dict):
+    """Get query as string for debug purpose"""
+    return psycopg.ClientCursor(conn).mogrify(query, params)

geovisio/utils/fields.py

@@ -38,6 +38,16 @@ class SortByField:
         revert_dir = SQLDirection.ASC if self.direction == SQLDirection.DESC else SQLDirection.DESC
         return sql.SQL("{column} {dir}").format(column=col, dir=revert_dir.value)
 
+    def as_non_aliased_sql(self) -> sql.Composable:
+        return sql.SQL("{column} {dir}").format(column=self.field.sql_column, dir=self.direction.value)
+
+    def revert_non_aliased_sql(self) -> sql.Composable:
+        revert_dir = SQLDirection.ASC if self.direction == SQLDirection.DESC else SQLDirection.DESC
+        return sql.SQL("{column} {dir}").format(column=self.field.sql_column, dir=revert_dir.value)
+
+    def as_stac(self) -> str:
+        return f"{'+' if self.direction == SQLDirection.ASC else '-'}{self.field.stac}"
+
 
 @dataclass
 class SortBy:
@@ -49,6 +59,18 @@ class SortBy:
     def revert(self) -> sql.Composable:
         return sql.SQL(", ").join([f.revert() for f in self.fields])
 
+    def as_non_aliased_sql(self) -> sql.Composable:
+        return sql.SQL(", ").join([f.as_non_aliased_sql() for f in self.fields])
+
+    def revert_non_aliased_sql(self) -> sql.Composable:
+        return sql.SQL(", ").join([f.revert_non_aliased_sql() for f in self.fields])
+
+    def as_stac(self) -> str:
+        return ",".join([f.as_stac() for f in self.fields])
+
+    def get_field_index(self, stac_name: str) -> int:
+        return next((i for i, f in enumerate(self.fields) if f.field.stac == stac_name))
+
 
 class Comparable(Protocol):
     """Protocol for annotating comparable types."""
@@ -63,13 +85,8 @@ T = TypeVar("T", bound=Comparable)
 class Bounds(Generic[T]):
     """Represent some bounds (min and max) over a generic type"""
 
-    min: T
-    max: T
-
-    def update(self, val: T):
-        """Update the bounds if val is out of them"""
-        self.min = min(self.min, val)
-        self.max = max(self.max, val)
+    first: T
+    last: T
 
 
 @dataclass

geovisio/utils/loggers.py

@@ -0,0 +1,14 @@
+import logging
+
+
+class LoggingWithExtra(logging.LoggerAdapter):
+    """Add some metadata to the log message"""
+
+    def process(self, msg, kwargs):
+        sep = " " if self.extra else ""
+        return f"{self.extra if self.extra else ''}{sep}{msg}", kwargs
+
+
+def getLoggerWithExtra(logger_name, extra):
+    """Create a logger with extra information. Those information will be displayed in the log message"""
+    return LoggingWithExtra(logging.getLogger(logger_name), extra)

geovisio/utils/model_query.py

@@ -9,11 +9,11 @@ class ParamsAndValues:
 
     params_as_dict: Dict[str, Any]
 
-    def __init__(self, model: BaseModel, **kwargs):
+    def __init__(self, model: BaseModel, jsonb_fields=set(), **kwargs):
         self.params_as_dict = model.model_dump(exclude_none=True) | kwargs
 
         for k, v in self.params_as_dict.items():
-            if isinstance(v, Dict):
+            if isinstance(v, Dict) or k in jsonb_fields:
                 self.params_as_dict[k] = Jsonb(v)  # convert dict to jsonb in database
 
     def has_updates(self):

geovisio/utils/params.py

@@ -12,9 +12,12 @@ def validation_error(e: ValidationError):
         }
         if d["input"]:
             detail["input"] = d["input"]
-            if "user_agent" in detail["input"]:
-                del detail["input"]["user_agent"]
-            if len(detail["input"]) == 0:
-                del detail["input"]
+            try:
+                if "user_agent" in detail["input"]:
+                    del detail["input"]["user_agent"]
+                if len(detail["input"]) == 0:
+                    del detail["input"]
+            except TypeError:
+                pass
         details.append(detail)
     return {"details": details}

geovisio/utils/pic_shape.py

@@ -0,0 +1,63 @@
+from typing import Annotated, Any, Iterator, List, Literal, NamedTuple, Tuple
+from pydantic import BaseModel, Discriminator, Field, Tag, field_validator
+
+BBox = Annotated[List[int], Field(min_length=4, max_length=4)]
+
+Position = NamedTuple("Position", [("x", int), ("y", int)])
+
+LinearRing = Annotated[List[Position], Field(min_length=4)]
+PolygonCoords = List[LinearRing]
+
+
+class Polygon(BaseModel):
+    type: Literal["Polygon"]
+    coordinates: PolygonCoords
+
+    @field_validator("coordinates")
+    def check_closure(cls, coordinates: List) -> List:
+        """Validate that Polygon is closed (first and last coordinate are the same)."""
+        if any(ring[-1] != ring[0] for ring in coordinates):
+            raise ValueError("All linear rings have the same start and end coordinates")
+
+        return coordinates
+
+    def coords_iter(self) -> Iterator[Tuple[int, int]]:
+        for ring in self.coordinates:
+            yield from ring
+
+    @classmethod
+    def from_bounds(cls, xmin: float, ymin: float, xmax: float, ymax: float) -> "Polygon":
+        """Create a Polygon geometry from a boundingbox."""
+        return cls(
+            type="Polygon",
+            coordinates=[[(xmin, ymin), (xmax, ymin), (xmax, ymax), (xmin, ymax), (xmin, ymin)]],
+        )
+
+
+Geometry = Polygon  # for the moment we support only polygons, but later we might add support for more, so the type is aliased
+
+
+def shape_discriminator(v: Any) -> str:
+    if isinstance(v, list):
+        return "bbox"
+    return "geometry"
+
+
+# Shapes can be provided as a bounding box or as a geometry
+InputAnnotationShape = Annotated[(Annotated[Polygon, Tag("geometry")] | Annotated[BBox, Tag("bbox")]), Discriminator(shape_discriminator)]
+
+
+def get_coords_from_shape(shape: InputAnnotationShape) -> Iterator[Tuple[int, int]]:
+    """Get an iterator to all coordinates of a shape"""
+    if shape_discriminator(shape) == "bbox":
+        yield shape[0], shape[1]
+        yield shape[2], shape[3]
+    else:
+        yield from shape.coords_iter()
+
+
+def shape_as_geometry(shape: InputAnnotationShape) -> Geometry:
+    """If the shape has been provided as a bounding box, transform it to a polygon"""
+    if shape_discriminator(shape) == "bbox":
+        return Polygon.from_bounds(*shape)
+    return shape

geovisio/utils/pictures.py

@@ -1,6 +1,8 @@
+import json
 import math
 from typing import Dict, Optional
 from uuid import UUID
+from attr import dataclass
 from flask import current_app, redirect, send_file
 from flask_babel import gettext as _
 import os
@@ -13,14 +15,24 @@ import logging
 from dataclasses import asdict
 from fs.path import dirname
 from psycopg.errors import UniqueViolation, InvalidParameterValue
+import sentry_sdk
 from geovisio import utils, errors
 from geopic_tag_reader import reader
 import re
+import multipart
 
 log = logging.getLogger(__name__)
 
 
-def createBlurredHDPicture(fs, blurApi, pictureBytes, outputFilename):
+@dataclass
+class BlurredPicture:
+    """Blurred picture's response"""
+
+    image: Image
+    metadata: Dict[str, str] = {}
+
+
+def createBlurredHDPicture(fs, blurApi, pictureBytes, outputFilename, keep_unblured_parts=False) -> Optional[BlurredPicture]:
     """Create the blurred version of a picture using a blurMask
 
     Parameters
@@ -40,19 +52,49 @@ def createBlurredHDPicture(fs, blurApi, pictureBytes, outputFilename):
             The blurred version of the image
     """
 
-    if blurApi is not None:
-        # Call blur API
-        pictureBytes.seek(0)
-        blurResponse = requests.post(blurApi + "/blur/", files={"picture": ("picture.jpg", pictureBytes.read(), "image/jpeg")})
-        blurResponse.raise_for_status()
-
-        # Save mask to FS
-        fs.writebytes(outputFilename, blurResponse.content)
+    if blurApi is None:
+        return None
+    # Call blur API, asking for multipart response if available
+    pictureBytes.seek(0)
+    query_params = {"keep": 1} if keep_unblured_parts else {}
+    blurResponse = requests.post(
+        f"{blurApi}/blur/",
+        files={"picture": ("picture.jpg", pictureBytes.read(), "image/jpeg")},
+        headers={"Accept": "multipart/form-data"},
+        params=query_params,
+    )
+    blurResponse.raise_for_status()
+
+    metadata, blurred_pic = None, None
+    content_type, content_type_params = multipart.parse_options_header(blurResponse.headers.get("content-type", ""))
+    if content_type == "multipart/form-data":
+        # New blurring api can return multipart response, with separated blurring picture/metadata
+        multipart_response = multipart.MultipartParser(io.BytesIO(blurResponse.content), boundary=content_type_params["boundary"])
+
+        metadata = multipart_response.get("metadata")
+        if metadata:
+            metadata = metadata.raw
+        blurred_pic = multipart_response.get("image")
+        if blurred_pic:
+            blurred_pic = blurred_pic.raw
+    else:
+        # old blurring API, no multipart response, we read the `x-sgblur` header
+        blurred_pic = blurResponse.content
+        metadata = blurResponse.headers.get("x-sgblur")
 
-        return Image.open(io.BytesIO(blurResponse.content))
+    # Save mask to FS
+    fs.writebytes(outputFilename, blurred_pic)
 
-    else:
-        return None
+    if metadata:
+        try:
+            metadata = json.loads(metadata)
+        except (json.decoder.JSONDecodeError, TypeError) as e:
+            # we skip the metadata's response if we are not able to understand it
+            log.warning(f"Impossible to parse blurring metadata API response: {e}")
+            sentry_sdk.capture_exception(e)
+            metadata = None
+
+    return BlurredPicture(image=Image.open(io.BytesIO(blurred_pic)), metadata=metadata)
 
 
 def getTileSize(imgSize):

geovisio/utils/reports.py

@@ -4,7 +4,7 @@ from typing import Optional, List
 from typing_extensions import Self
 from datetime import datetime
 from pydantic import BaseModel, ConfigDict
-from geovisio.utils import db
+from geovisio.utils import cql2, db
 from geovisio.errors import InvalidAPIUsage
 from flask import current_app
 from psycopg.sql import SQL
@@ -103,6 +103,13 @@ def is_picture_owner(report: Report, account_id: UUID):
     return isOwner
 
 
+REPORT_FILTER_TO_DB_FIELDS = {
+    "status": "r.status",
+    "reporter": "reporter_account_id",
+    "owner": "COALESCE(p.account_id, s.account_id)",
+}
+
+
 def _parse_filter(filter: Optional[str]) -> SQL:
     """
     Parse a filter string and return a SQL expression
@@ -124,22 +131,8 @@ def _parse_filter(filter: Optional[str]) -> SQL:
     """
     if not filter:
         return SQL("TRUE")
-    from pygeofilter.backends.sql import to_sql_where
-    from pygeofilter.parsers.ecql import parse as ecql_parser
-
-    try:
-        filterAst = ecql_parser(filter)
-        fieldsToFilter = {
-            "status": "r.status",
-            "reporter": "reporter_account_id",
-            "owner": "COALESCE(p.account_id, s.account_id)",
-        }
-
-        f = to_sql_where(filterAst, fieldsToFilter).replace('"', "").replace("'me'", "%(account_id)s")  # type: ignore
-        return SQL(f)  # type: ignore
-    except Exception as e:
-        print(e)
-        raise InvalidAPIUsage(_("Unsupported filter parameter"), status_code=400)
+
+    return cql2.parse_cql2_filter(filter, REPORT_FILTER_TO_DB_FIELDS)
 
 
 def list_reports(account_id: UUID, limit: int = 100, filter: Optional[str] = None, forceAccount: bool = True) -> Reports: