geovisio 2.9.0__py3-none-any.whl → 2.10.0__py3-none-any.whl

geovisio/translations/zh_Hant/LC_MESSAGES/messages.po

@@ -308,7 +308,7 @@ msgstr ""
 
 #: geovisio/web/items.py:420
 #, python-format
-msgid "Picture with id %(p)s does not exists"
+msgid "Picture with id %(p)s does not exist"
 msgstr ""
 
 #: geovisio/web/items.py:706

geovisio/utils/annotations.py

@@ -62,31 +62,28 @@ Note that the API will always output geometry as geojson geometry (thus will tra
         return shape_as_geometry(self.shape)
 
 
-def creation_annotation(params: AnnotationCreationParameter) -> Annotation:
-    """Create an annotation in the database"""
+def creation_annotation(params: AnnotationCreationParameter, conn: psycopg.Connection) -> Annotation:
+    """Create an annotation in the database.
+    Note, this should be called from an autocommit connection"""
 
     model = model_query.get_db_params_and_values(
         AnnotationCreationRow(picture_id=params.picture_id, shape=params.shape_as_geometry()), jsonb_fields={"shape"}
     )
-    insert_query = SQL(
-        """WITH existing_annotations AS (
-        SELECT * FROM annotations WHERE picture_id = %(picture_id)s AND shape = %(shape)s
-)
-, new_ones AS (
-        INSERT INTO annotations (picture_id, shape)
-        SELECT %(picture_id)s, %(shape)s
-        WHERE NOT EXISTS (SELECT FROM existing_annotations)
-        RETURNING *
-)
-SELECT * FROM existing_annotations UNION ALL SELECT * FROM new_ones 
-;"""
-    )
 
-    with db.conn(current_app) as conn, conn.transaction(), conn.cursor(row_factory=class_row(Annotation)) as cursor:
+    with conn.transaction(), conn.cursor(row_factory=class_row(Annotation)) as cursor:
         # we check that the shape is valid
         check_shape(conn, params)
 
-        annotation = cursor.execute(insert_query, model.params_as_dict).fetchone()
+        annotation = cursor.execute(
+            "SELECT * FROM annotations WHERE picture_id = %(picture_id)s AND shape = %(shape)s", model.params_as_dict
+        ).fetchone()
+        if annotation is None:
+            annotation = cursor.execute(
+                """INSERT INTO annotations (picture_id, shape)
+        VALUES (%(picture_id)s, %(shape)s)
+        RETURNING *""",
+                model.params_as_dict,
+            ).fetchone()
 
         if annotation is None:
             raise Exception("Impossible to insert annotation in database")

geovisio/utils/auth.py

@@ -31,7 +31,7 @@ class OAuthUserAccount(object):
 
 
 class OAuthProvider(ABC):
-    """Base class for oauth provider. Need so specify how to get user's info"""
+    """Base class for oauth provider. Need to specify how to get user's info"""
 
     name: str
     client: Any
@@ -52,7 +52,7 @@ class OAuthProvider(ABC):
         """
         URL to a user settings page.
         This URL should point to a web page where user can edit its password or email address,
-        if that makes sense regardinz your GeoVisio instance.
+        if that makes sense regarding your GeoVisio instance.
 
         This is useful if your instance has its own specific identity provider. It may not be used if you rely on third-party auth provider.
         """
@@ -235,7 +235,7 @@ class Account(BaseModel):
 
 
 def account_allow_collaborative_editing(account_id: str | UUID):
-    """An account allow collaborative editing it if has been allow at the account level else we check the instance configuration"""
+    """An account allows collaborative editing it if has been allowed at the account level else we check the instance configuration"""
     r = db.fetchone(
         current_app,
         """SELECT COALESCE(accounts.collaborative_metadata, configurations.collaborative_metadata, true) AS collaborative_metadata
@@ -249,15 +249,16 @@ WHERE accounts.id = %s""",
 
 
 def login_required():
-    """Check that the user is logged, and abort if it's not the case"""
+    """Check that the user is logged in, and abort if it's not the case"""
 
     def actual_decorator(f):
         @wraps(f)
         def decorator(*args, **kwargs):
-            account = get_current_account()
-            if not account:
-                return flask.abort(flask.make_response(flask.jsonify(message=_("Authentication is mandatory")), 401))
-            kwargs["account"] = account
+            if "account" not in kwargs:
+                account = get_current_account()
+                if not account:
+                    return flask.abort(flask.make_response(flask.jsonify(message=_("Authentication is mandatory")), 401))
+                kwargs["account"] = account
 
             return f(*args, **kwargs)
 
@@ -267,7 +268,7 @@ def login_required():
 
 
 def login_required_by_setting(mandatory_login_param):
-    """Check that the user is logged, and abort if it's not the case
+    """Check that the user is logged in, and abort if it's not the case
 
     Args:
             mandatory_login_param (str): name of the configuration parameter used to decide if the login is mandatory or not
@@ -303,7 +304,7 @@ def login_required_by_setting(mandatory_login_param):
 
 
 def login_required_with_redirect():
-    """Check that the user is logged, and redirect if it's not the case"""
+    """Check that the user is logged in, and redirect if it's not the case"""
 
     def actual_decorator(f):
         @wraps(f)
@@ -346,7 +347,7 @@ class UnknowAccountException(Exception):
     status_code = 401
 
     def __init__(self):
-        msg = "No account with this oauth id is know, you should login first"
+        msg = "No account with this oauth id is known, you should login first"
         super().__init__(msg)
 
 
@@ -358,12 +359,12 @@ class LoginRequiredException(Exception):
         super().__init__(msg)
 
 
-def get_current_account():
+def get_current_account() -> Optional[Account]:
     """Get the authenticated account information.
 
     This account is either stored in the flask's session or retrieved with the Bearer token passed with an `Authorization` header.
 
-    The flask session is usually used by browser, whereas the bearer token is handly for non interactive uses, like curls or CLI usage.
+    The flask session is usually used by browser, whereas the bearer token is handy for non interactive uses, like curls or CLI usage.
 
     Returns:
                     Account: the current logged account, None if nobody is logged

geovisio/utils/cql2.py

@@ -66,12 +66,12 @@ def parse_search_filter(value: Optional[str]) -> Optional[sql.SQL]:
 UNION
     SELECT DISTINCT(picture_id)
     FROM annotations_semantics ans
-    JOIN annotations a on a.id = ans.annotation_id
+    JOIN annotations a ON a.id = ans.annotation_id
     WHERE {semantic_filter}
 UNION
     SELECT sp.pic_id
     FROM sequences_pictures sp
-    join sequences_semantics sm on sp.seq_id = sm.sequence_id
+    JOIN sequences_semantics sm ON sp.seq_id = sm.sequence_id 
     WHERE {semantic_filter}
 LIMIT %(limit)s
 ))"""

geovisio/utils/fields.py

@@ -2,6 +2,8 @@ from enum import Enum
 from dataclasses import dataclass, field
 from typing import Any, List, Generic, TypeVar, Protocol
 from psycopg import sql
+from geovisio import errors
+from gettext import gettext as _
 
 
 @dataclass
@@ -12,8 +14,8 @@ class FieldMapping:
     stac: str
 
     @property
-    def sql_filter(self) -> sql.Composable:
-        return sql.SQL("s.{}").format(self.sql_column)
+    def sql_filter(self, row_alias="s.") -> sql.Composable:
+        return sql.SQL(row_alias + "{}").format(self.sql_column)
 
 
 class SQLDirection(Enum):
@@ -97,3 +99,13 @@ class BBox:
     maxx: float
     miny: float
     maxy: float
+
+
+def parse_relative_heading(value: str) -> int:
+    try:
+        relHeading = int(value)
+        if relHeading < -180 or relHeading > 180:
+            raise ValueError()
+        return relHeading
+    except (ValueError, TypeError):
+        raise errors.InvalidAPIUsage(_("Relative heading is not valid, should be an integer in degrees from -180 to 180"), status_code=400)

geovisio/utils/items.py

@@ -0,0 +1,44 @@
+from .fields import SQLDirection
+from psycopg.sql import SQL, Identifier
+from enum import Enum
+from dataclasses import dataclass, field
+from typing import Optional, List
+
+
+class SortableItemField(Enum):
+    ts = Identifier("ts")
+    updated = Identifier("updated_at")
+    distance_to = ""
+    id = Identifier("id")
+
+
+@dataclass
+class ItemSortByField:
+    field: SortableItemField
+    direction: SQLDirection
+
+    # Note that this obj_to_compare is only used for the `distance_to` field, but we cannot put it in the enum
+    obj_to_compare: Optional[SQL] = None
+
+    def to_sql(self, alias) -> SQL:
+        sql_order = None
+        if self.obj_to_compare:
+            if self.field == SortableItemField.distance_to:
+                sql_order = SQL('{alias}."geom" <-> {obj_to_compare} {direction}').format(
+                    alias=alias, obj_to_compare=self.obj_to_compare, direction=self.direction.value
+                )
+            else:
+                raise InvalidAPIUsage("For the moment only the distance comparison to another item is supported")
+        else:
+            sql_order = SQL("{alias}.{field} {direction}").format(alias=alias, field=self.field.value, direction=self.direction.value)
+        return sql_order
+
+
+@dataclass
+class SortBy:
+    fields: List[ItemSortByField] = field(default_factory=lambda: [])
+
+    def to_sql(self, alias=Identifier("p")) -> SQL:
+        if len(self.fields) == 0:
+            return SQL("")
+        return SQL("ORDER BY {fields}").format(fields=SQL(", ").join([f.to_sql(alias=alias) for f in self.fields]))

geovisio/utils/model_query.py

@@ -28,7 +28,7 @@ class ParamsAndValues:
         return SQL(", ").join([Placeholder(f) for f in self.params_as_dict.keys()])
 
     def fields_for_set(self) -> Composed:
-        """Get the fields and the placeholders formated for an update query like:
+        """Get the fields and the placeholders formatted for an update query like:
         '"a" = %(a)s, "b" = %(b)s'
 
         Can be used directly with a query like:
@@ -39,7 +39,7 @@ class ParamsAndValues:
         return SQL(", ").join(self.fields_for_set_list())
 
     def fields_for_set_list(self) -> List[Composed]:
-        """Get the fields and the placeholders formated for an update query like:
+        """Get the fields and the placeholders formatted for an update query like:
         ['"a" = %(a)s', '"b" = %(b)s']
 
         Note that the returned list should be joined with SQL(", ").join()

geovisio/utils/pic_shape.py

@@ -15,7 +15,7 @@ class Polygon(BaseModel):
 
     @field_validator("coordinates")
     def check_closure(cls, coordinates: List) -> List:
-        """Validate that Polygon is closed (first and last coordinate are the same)."""
+        """Validate that Polygon is closed (first and last coordinates are the same)."""
         if any(ring[-1] != ring[0] for ring in coordinates):
             raise ValueError("All linear rings have the same start and end coordinates")
 

geovisio/utils/pictures.py

@@ -1,6 +1,6 @@
 import json
 import math
-from typing import Dict, Optional
+from typing import Any, Dict, List, Optional, Tuple
 from uuid import UUID
 from attr import dataclass
 from flask import current_app, redirect, send_file
@@ -15,12 +15,16 @@ import logging
 from dataclasses import asdict
 from fs.path import dirname
 from psycopg.errors import UniqueViolation, InvalidParameterValue
+from psycopg.types.json import Jsonb
+from psycopg import sql, Connection
 import sentry_sdk
 from geovisio import utils, errors
 from geopic_tag_reader import reader
 import re
 import multipart
 
+from geovisio.utils import db
+
 log = logging.getLogger(__name__)
 
 
@@ -51,7 +55,6 @@ def createBlurredHDPicture(fs, blurApi, pictureBytes, outputFilename, keep_unblu
     PIL.Image
             The blurred version of the image
     """
-
     if blurApi is None:
         return None
     # Call blur API, asking for multipart response if available
@@ -553,6 +556,25 @@ class MetadataReadingError(Exception):
         self.missing_mandatory_tags = missing_mandatory_tags
 
 
+def get_lighter_metadata(metadata):
+    """Create a lighter metadata field to remove duplicates fields"""
+    lighterMetadata = dict(
+        filter(
+            lambda v: v[0] not in ["ts", "heading", "lon", "lat", "exif", "originalContentMd5", "ts_by_source", "gps_accuracy"],
+            metadata.items(),
+        )
+    )
+    if lighterMetadata.get("tagreader_warnings") is not None and len(lighterMetadata["tagreader_warnings"]) == 0:
+        del lighterMetadata["tagreader_warnings"]
+    lighterMetadata["tz"] = metadata["ts"].tzname()
+    if metadata.get("ts_by_source", {}).get("gps") is not None:
+        lighterMetadata["ts_gps"] = metadata["ts_by_source"]["gps"].isoformat()
+    if metadata.get("ts_by_source", {}).get("camera") is not None:
+        lighterMetadata["ts_camera"] = metadata["ts_by_source"]["camera"].isoformat()
+
+    return lighterMetadata
+
+
 def insertNewPictureInDatabase(
     db, sequenceId, position, pictureBytes, associatedAccountID, additionalMetadata, uploadSetID=None, lang="en"
 ):
@@ -579,11 +601,10 @@ def insertNewPictureInDatabase(
     -------
     uuid : The uuid of the new picture entry in the database
     """
-    from psycopg.types.json import Jsonb
 
     # Create a fully-featured metadata object
-    picturePillow = Image.open(io.BytesIO(pictureBytes))
-    metadata = readPictureMetadata(pictureBytes, lang) | utils.pictures.getPictureSizing(picturePillow) | additionalMetadata
+    with Image.open(io.BytesIO(pictureBytes)) as picturePillow:
+        metadata = readPictureMetadata(pictureBytes, lang) | utils.pictures.getPictureSizing(picturePillow) | additionalMetadata
 
     # Remove cols/rows information for flat pictures
     if metadata["type"] == "flat":
@@ -591,19 +612,7 @@ def insertNewPictureInDatabase(
         metadata.pop("rows")
 
     # Create a lighter metadata field to remove duplicates fields
-    lighterMetadata = dict(
-        filter(
-            lambda v: v[0] not in ["ts", "heading", "lon", "lat", "exif", "originalContentMd5", "ts_by_source", "gps_accuracy"],
-            metadata.items(),
-        )
-    )
-    if lighterMetadata.get("tagreader_warnings") is not None and len(lighterMetadata["tagreader_warnings"]) == 0:
-        del lighterMetadata["tagreader_warnings"]
-    lighterMetadata["tz"] = metadata["ts"].tzname()
-    if metadata.get("ts_by_source", {}).get("gps") is not None:
-        lighterMetadata["ts_gps"] = metadata["ts_by_source"]["gps"].isoformat()
-    if metadata.get("ts_by_source", {}).get("camera") is not None:
-        lighterMetadata["ts_camera"] = metadata["ts_by_source"]["camera"].isoformat()
+    lighterMetadata = get_lighter_metadata(metadata)
 
     exif = cleanupExif(metadata["exif"])
 
@@ -639,6 +648,90 @@ def insertNewPictureInDatabase(
     return picId
 
 
+def _get_metadata_to_update(db_picture: Dict, new_reader_metadata: reader.GeoPicTags) -> Tuple[List[str], Dict[str, Any]]:
+    fields_to_update = []
+    params = {}
+
+    if new_reader_metadata.ts != db_picture["ts"]:
+        fields_to_update.append(sql.SQL("ts = %(ts)s"))
+        params["ts"] = new_reader_metadata.ts.isoformat()
+    if db_picture["heading_computed"] is False and new_reader_metadata.heading != db_picture["heading"]:
+        fields_to_update.append(sql.SQL("heading = %(heading)s"))
+        params["heading"] = new_reader_metadata.heading
+    if new_reader_metadata.gps_accuracy != db_picture["gps_accuracy_m"]:
+        fields_to_update.append(sql.SQL("gps_accuracy_m = %(gps_accuracy_m)s"))
+        params["gps_accuracy_m"] = new_reader_metadata.gps_accuracy
+
+    # Note: The db metadata can have more stuff (like originalFileName, size, ...), we so only check if the new value is different from the old one
+    # we cannot check directly for dict equality
+    new_lighterMetadata = get_lighter_metadata(asdict(new_reader_metadata))
+    metadata_updates = {}
+    for k, v in new_lighterMetadata.items():
+        if v != db_picture["metadata"].get(k):
+            metadata_updates[k] = v
+
+    # if the position has been updated (by more than ~10cm)
+    lon, lat = db_picture["lon"], db_picture["lat"]
+    new_lon, new_lat = new_reader_metadata.lon, new_reader_metadata.lat
+    if not math.isclose(lon, new_lon, abs_tol=0.0000001) or not math.isclose(lat, new_lat, abs_tol=0.0000001):
+        fields_to_update.append(sql.SQL("geom = ST_SetSRID(ST_MakePoint(%(lon)s, %(lat)s), 4326)"))
+        params["lon"] = new_reader_metadata.lon
+        params["lat"] = new_reader_metadata.lat
+
+    if metadata_updates:
+        fields_to_update.append(sql.SQL("metadata = metadata || %(new_metadata)s"))
+        params["new_metadata"] = Jsonb(metadata_updates)
+
+    return fields_to_update, params
+
+
+def ask_for_metadata_update(picture_id: UUID, read_file=False):
+    """Enqueue an async job to reread the picture's metadata"""
+    args = Jsonb({"read_file": True}) if read_file else None
+    with db.conn(current_app) as conn:
+        conn.execute(
+            "INSERT INTO job_queue(picture_id, task, args) VALUES (%s, 'read_metadata', %s)",
+            [picture_id, args],
+        )
+
+
+def update_picture_metadata(conn: Connection, picture_id: UUID, read_file=False) -> bool:
+    """Update picture metadata in database, using either the stored metadata or the original file
+
+    Only updates metadata that have changed.
+    Returns True if some metadata have been updated, False otherwise
+    """
+
+    with conn.cursor(row_factory=dict_row) as cursor:
+        db_picture = cursor.execute(
+            "SELECT ts, heading, metadata, ST_X(geom) as lon, ST_Y(geom) as lat, account_id, exif, gps_accuracy_m, heading_computed FROM pictures WHERE id = %s",
+            [picture_id],
+        ).fetchone()
+        if db_picture is None:
+            raise Exception(f"Picture {picture_id} not found")
+
+    if read_file:
+        pic_path = getHDPicturePath(picture_id)
+
+        with current_app.config["FILESYSTEMS"].permanent.openbin(pic_path) as picture_bytes:
+            new_metadata = reader.readPictureMetadata(picture_bytes.read())
+    else:
+        new_metadata = reader.getPictureMetadata(db_picture["exif"], db_picture["metadata"]["width"], db_picture["metadata"]["height"])
+
+    # we want to only updates values that have changed
+    fields_to_update, params = _get_metadata_to_update(db_picture, new_metadata)
+
+    if not fields_to_update:
+        logging.debug(f"No metadata update needed for picture {picture_id}")
+        return False
+
+    conn.execute(
+        sql.SQL("UPDATE pictures SET {f} WHERE id = %(pic_id)s").format(f=sql.SQL(", ").join(fields_to_update)),
+        params | {"pic_id": picture_id},
+    )
+    return True
+
+
 # Note: we don't want to store and expose exif binary fields as they are difficult to use and take a lot of storage in the database (~20% for maker notes only)
 # This list has been queried from real data (cf [this comment](https://gitlab.com/panoramax/server/api/-/merge_requests/241#note_1790580636)).
 # Update this list (and do a sql migration) if new binary fields are added

geovisio/utils/semantics.py

@@ -22,6 +22,7 @@ class EntityType(Enum):
     pic = "picture_id"
     seq = "sequence_id"
     annotation = "annotation_id"
+    upload_set = "upload_set_id"
 
 
 @dataclass
@@ -37,10 +38,12 @@ class Entity:
                 return Identifier("sequences_semantics")
             case EntityType.annotation:
                 return Identifier("annotations_semantics")
+            case EntityType.upload_set:
+                return Identifier("upload_sets_semantics")
             case _:
                 raise ValueError(f"Unknown entity type: {self.type}")
 
-    def get_history_table(self) -> Identifier:
+    def get_history_table(self) -> Optional[Identifier]:
         match self.type:
             case EntityType.pic:
                 return Identifier("pictures_semantics_history")
@@ -48,6 +51,8 @@ class Entity:
                 return Identifier("sequences_semantics_history")
             case EntityType.annotation:
                 return Identifier("pictures_semantics_history")
+            case EntityType.upload_set:
+                return None
             case _:
                 raise ValueError(f"Unknown entity type: {self.type}")
 
@@ -77,14 +82,34 @@ AND ({filter})"""
                 params,
             )
         if tag_to_add:
+            fields = [Identifier(entity.type.value), Identifier("key"), Identifier("value")]
+            if entity.type == EntityType.upload_set:
+                # upload_set semantics have no history, the account is directly stored in the table
+                fields.append(Identifier("account_id"))
+
             with cursor.copy(
                 SQL("COPY {table} ({fields}) FROM STDIN").format(
                     table=table_name,
-                    fields=SQL(",").join([Identifier(entity.type.value), Identifier("key"), Identifier("value")]),
+                    fields=SQL(",").join(fields),
                 )
             ) as copy:
                 for tag in tag_to_add:
-                    copy.write_row((entity.id, tag.key, tag.value))
+                    row = [entity.id, tag.key, tag.value]
+                    if entity.type == EntityType.upload_set:
+                        row.append(account)
+                    copy.write_row(row)
+        if tag_to_delete and entity.type == EntityType.annotation and not tag_to_add:
+            # if tags have been deleted, we check if some annotations are now empty and need to be deleted
+            cursor.execute(
+                """DELETE FROM annotations
+WHERE id  = %(annotation_id)s AND 
+(
+    SELECT count(*) AS nb_semantics 
+    FROM annotations_semantics 
+    WHERE annotation_id = %(annotation_id)s
+) = 0""",
+                {"annotation_id": entity.id},
+            )
         if tag_to_add or tag_to_delete:
             # we track the history changes of the semantic tags
             track_semantic_history(cursor, entity, actions, account, annotation)
@@ -100,6 +125,10 @@ class SemanticTagUpdateOnAnnotation(SemanticTagUpdate):
 
 
 def track_semantic_history(cursor: Cursor, entity: Entity, actions: List[SemanticTagUpdate], account: UUID, annotation):
+    history_table = entity.get_history_table()
+    if history_table is None:
+        # no history for upload_set semantics
+        return
     params = {
         "account_id": account,
     }

geovisio/utils/sentry.py

@@ -73,7 +73,7 @@ def _wrap_cursor_execute(f):
 
 
 class FileSystemIntegration(Integration):
-    """Add metrics to the 2 most usefull filesystem, the 'os file' filesystem and the s3 filesystem"""
+    """Add metrics to the 2 most useful filesystem, the 'os file' filesystem and the s3 filesystem"""
 
     identifier = "filesystem"