geovisio 2.7.1__py3-none-any.whl → 2.8.1__py3-none-any.whl

geovisio/utils/auth.py

@@ -1,3 +1,6 @@
+from ast import Dict
+from uuid import UUID
+from click import Option
 import flask
 from flask import current_app, url_for, session, redirect, request
 from flask_babel import gettext as _
@@ -8,9 +11,10 @@ from abc import ABC, abstractmethod
 from typing import Any
 from typing import Optional
 from enum import Enum
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
 import sentry_sdk
 from psycopg.rows import dict_row
+from geovisio import errors
 from geovisio.utils import db
 
 
@@ -159,16 +163,30 @@ class Account(BaseModel):
     name: str
     oauth_provider: Optional[str] = None
     oauth_id: Optional[str] = None
-
-    model_config = ConfigDict(extra="forbid")
+    tos_accepted: Optional[bool] = None
 
     def __init__(self, role: Optional[AccountRole] = None, **kwargs) -> None:
+        # Note: since it's a valid state for the collaborative_metadata to be None,
+        # we need to only set it if provided, this way we can check the `model_fields_set` to know if the collaborative_metadata is set
+        collaborative_metadata_set = "collaborative_metadata" in kwargs
+        collaborative_metadata = kwargs.pop("collaborative_metadata", None)
         super().__init__(**kwargs)
         self.role = role
+        if collaborative_metadata_set:
+            self.collaborative_metadata = collaborative_metadata
 
-    # Note: this field is excluded since we do not want to persist it in the cookie. It will be fetched from the database if needed
-    # and accessed though the `role` property
+    # Note: those fields are excluded since we do not want to persist it in the cookie. It will be fetched from the database if needed
     role_: Optional[AccountRole] = Field(default=None, exclude=True)
+    collaborative_metadata_: Optional[bool] = Field(default=None, exclude=True)
+
+    @field_validator("id", mode="before")
+    @classmethod
+    def check_id(cls, value) -> str:
+        if isinstance(value, UUID):
+            return str(value)
+        if isinstance(value, str):
+            return value
+        raise ValidationError("Invalid account id type")
 
     def can_check_reports(self):
         """Is account legitimate to read any report ?"""
@@ -178,17 +196,57 @@ class Account(BaseModel):
         """Is account legitimate to read and edit excluded areas ?"""
         return self.role == AccountRole.admin
 
+    def can_edit_pages(self):
+        """Is account legitimate to edit web pages ?"""
+        return self.role == AccountRole.admin
+
     @property
     def role(self) -> AccountRole:
         if self.role_ is None:
-            role = db.fetchone(current_app, "SELECT role FROM accounts WHERE id = %s", (self.id,), row_factory=dict_row)
-            self.role_ = AccountRole(role["role"])
+            self._fetch_database_info()
         return self.role_
 
     @role.setter
-    def role(self, r: AccountRole) -> None:
+    def role(self, r: AccountRole | str) -> None:
+        if isinstance(r, str):
+            r = AccountRole(r)
         self.role_ = r
 
+    @property
+    def collaborative_metadata(self) -> Optional[bool]:
+        if "collaborative_metadata_" not in self.model_fields_set:
+            self._fetch_database_info()
+        return self.collaborative_metadata_
+
+    @collaborative_metadata.setter
+    def collaborative_metadata(self, b: Optional[bool]) -> None:
+        self.collaborative_metadata_ = b
+
+    def _fetch_database_info(self):
+        """Fetch the missing database metadata for this account"""
+        r = db.fetchone(
+            current_app,
+            "SELECT role, collaborative_metadata FROM accounts WHERE id = %s",
+            (self.id,),
+            row_factory=dict_row,
+        )
+        self.role = AccountRole(r["role"])
+        self.collaborative_metadata = r["collaborative_metadata"]
+
+
+def account_allow_collaborative_editing(account_id: str | UUID):
+    """An account allow collaborative editing it if has been allow at the account level else we check the instance configuration"""
+    r = db.fetchone(
+        current_app,
+        """SELECT COALESCE(accounts.collaborative_metadata, configurations.collaborative_metadata, true) AS collaborative_metadata
+FROM accounts
+JOIN configurations ON TRUE
+WHERE accounts.id = %s""",
+        [account_id],
+        row_factory=dict_row,
+    )
+    return r["collaborative_metadata"]
+
 
 def login_required():
     """Check that the user is logged, and abort if it's not the case"""
@@ -221,6 +279,20 @@ def login_required_by_setting(mandatory_login_param):
             account = get_current_account()
             if not account and current_app.config[mandatory_login_param]:
                 return flask.abort(flask.make_response(flask.jsonify(message="Authentication is mandatory"), 401))
+            if account and account.tos_accepted is False and current_app.config["API_ENFORCE_TOS_ACCEPTANCE"]:
+                tos_acceptance_page = current_app.config["API_WEBSITE_URL"].tos_validation_page()
+                raise errors.InvalidAPIUsage(
+                    message=_(
+                        "You need to accept the terms of service before uploading any pictures. You can do so by validating them here: %(url)s",
+                        url=tos_acceptance_page,
+                    ),
+                    status_code=401,
+                    payload={
+                        "details": {
+                            "validation_page": tos_acceptance_page,
+                        }
+                    },
+                )
             kwargs["account"] = account
 
             return f(*args, **kwargs)

geovisio/utils/link.py

@@ -6,9 +6,10 @@ from flask import url_for
 class Link(BaseModel):
     rel: str
     type: str
-    title: Optional[str]
+    title: Optional[str] = None
     href: str
 
 
 def make_link(rel: str, route: str, title: Optional[str] = None, type: str = "application/json", **args):
-    return Link(rel=rel, type=type, title=title, href=url_for(route, **args, _external=True))
+    kwargs = {"title": title} if title else {}  # do not pass none title, to know if it has been set or not
+    return Link(rel=rel, type=type, href=url_for(route, **args, _external=True), **kwargs)

geovisio/utils/loggers.py

@@ -0,0 +1,14 @@
+import logging
+
+
+class LoggingWithExtra(logging.LoggerAdapter):
+    """Add some metadata to the log message"""
+
+    def process(self, msg, kwargs):
+        sep = " " if self.extra else ""
+        return f"{self.extra if self.extra else ''}{sep}{msg}", kwargs
+
+
+def getLoggerWithExtra(logger_name, extra):
+    """Create a logger with extra information. Those information will be displayed in the log message"""
+    return LoggingWithExtra(logging.getLogger(logger_name), extra)

geovisio/utils/model_query.py

@@ -0,0 +1,55 @@
+from typing import Any, Dict, List
+from pydantic import BaseModel
+from psycopg.sql import SQL, Identifier, Placeholder, Composed
+from psycopg.types.json import Jsonb
+
+
+class ParamsAndValues:
+    """Simple wrapper used to help building a query with the right psycopg types"""
+
+    params_as_dict: Dict[str, Any]
+
+    def __init__(self, model: BaseModel, **kwargs):
+        self.params_as_dict = model.model_dump(exclude_none=True) | kwargs
+
+        for k, v in self.params_as_dict.items():
+            if isinstance(v, Dict):
+                self.params_as_dict[k] = Jsonb(v)  # convert dict to jsonb in database
+
+    def has_updates(self):
+        return bool(self.params_as_dict)
+
+    def fields(self) -> Composed:
+        """Get the database fields identifiers"""
+        return SQL(", ").join([Identifier(f) for f in self.params_as_dict.keys()])
+
+    def placeholders(self) -> Composed:
+        """Get the placeholders for the query"""
+        return SQL(", ").join([Placeholder(f) for f in self.params_as_dict.keys()])
+
+    def fields_for_set(self) -> Composed:
+        """Get the fields and the placeholders formated for an update query like:
+        '"a" = %(a)s, "b" = %(b)s'
+
+        Can be used directly with a query like:
+        ```python
+        SQL("UPDATE some_table SET {fields}").format(fields=fields)
+        ```
+        """
+        return SQL(", ").join(self.fields_for_set_list())
+
+    def fields_for_set_list(self) -> List[Composed]:
+        """Get the fields and the placeholders formated for an update query like:
+        ['"a" = %(a)s', '"b" = %(b)s']
+
+        Note that the returned list should be joined with SQL(", ").join()
+        """
+        return [SQL("{f} = {p}").format(f=Identifier(f), p=Placeholder(f)) for f in self.params_as_dict.keys()]
+
+
+def get_db_params_and_values(model: BaseModel, **kwargs):
+    """Get a simple wrapper to help building a query with the right psycopg types
+
+    check the unit tests in test_model_query.py for examples
+    """
+    return ParamsAndValues(model, **kwargs)

geovisio/utils/params.py

@@ -12,9 +12,12 @@ def validation_error(e: ValidationError):
         }
         if d["input"]:
             detail["input"] = d["input"]
-            if "user_agent" in detail["input"]:
-                del detail["input"]["user_agent"]
-            if len(detail["input"]) == 0:
-                del detail["input"]
+            try:
+                if "user_agent" in detail["input"]:
+                    del detail["input"]["user_agent"]
+                if len(detail["input"]) == 0:
+                    del detail["input"]
+            except TypeError:
+                pass
         details.append(detail)
     return {"details": details}

geovisio/utils/pictures.py

@@ -550,7 +550,10 @@ def insertNewPictureInDatabase(
 
     # Create a lighter metadata field to remove duplicates fields
     lighterMetadata = dict(
-        filter(lambda v: v[0] not in ["ts", "heading", "lon", "lat", "exif", "originalContentMd5", "ts_by_source"], metadata.items())
+        filter(
+            lambda v: v[0] not in ["ts", "heading", "lon", "lat", "exif", "originalContentMd5", "ts_by_source", "gps_accuracy"],
+            metadata.items(),
+        )
     )
     if lighterMetadata.get("tagreader_warnings") is not None and len(lighterMetadata["tagreader_warnings"]) == 0:
         del lighterMetadata["tagreader_warnings"]
@@ -566,11 +569,9 @@ def insertNewPictureInDatabase(
         # Add picture metadata to database
         try:
             picId = db.execute(
-                """
-                INSERT INTO pictures (ts, heading, metadata, geom, account_id, exif, original_content_md5, upload_set_id)
-                VALUES (%s, %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326), %s, %s, %s, %s)
-                RETURNING id
-            """,
+                """INSERT INTO pictures (ts, heading, metadata, geom, account_id, exif, original_content_md5, upload_set_id, gps_accuracy_m)
+                VALUES (%s, %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326), %s, %s, %s, %s, %s)
+                RETURNING id""",
                 (
                     metadata["ts"].isoformat(),
                     metadata["heading"],
@@ -581,45 +582,12 @@ def insertNewPictureInDatabase(
                     Jsonb(exif),
                     metadata.get("originalContentMd5"),
                     uploadSetID,
+                    metadata.get("gps_accuracy"),
                 ),
             ).fetchone()[0]
         except InvalidParameterValue as e:
             raise InvalidMetadataValue(e.diag.message_primary) from e
 
-        # Process field of view for each pictures
-        # Flat pictures = variable fov
-        if metadata["type"] == "flat":
-            make, model = metadata.get("make"), metadata.get("model")
-            if make is not None and model is not None and metadata["focal_length"] != 0:
-                db.execute("SET pg_trgm.similarity_threshold = 0.9")
-                db.execute(
-                    """
-					UPDATE pictures
-					SET metadata = jsonb_set(metadata, '{field_of_view}'::text[], COALESCE(
-						(
-							SELECT ROUND(DEGREES(2 * ATAN(sensor_width / (2 * (metadata->>'focal_length')::float))))::varchar
-							FROM cameras
-							WHERE model %% CONCAT(%(make)s::text, ' ', %(model)s::text)
-							ORDER BY model <-> CONCAT(%(make)s::text, ' ', %(model)s::text)
-							LIMIT 1
-						),
-						'null'
-					)::jsonb)
-					WHERE id = %(id)s
-				""",
-                    {"id": picId, "make": make, "model": model},
-                )
-
-        # 360 pictures = 360° fov
-        else:
-            db.execute(
-                """
-				UPDATE pictures
-				SET metadata = jsonb_set(metadata, '{field_of_view}'::text[], '360'::jsonb)
-    			WHERE id = %s
-			""",
-                [picId],
-            )
         if sequenceId is not None:
             try:
                 db.execute("INSERT INTO sequences_pictures(seq_id, rank, pic_id) VALUES(%s, %s, %s)", [sequenceId, position, picId])
@@ -681,11 +649,11 @@ def readPictureMetadata(picture: bytes, lang: Optional[str] = "en") -> dict:
         try:
             if isinstance(v, bytes):
                 try:
-                    cleanedExif[k] = v.decode("utf-8").replace("\x00", "")
+                    cleanedExif[k] = v.decode("utf-8").replace("\x00", "").replace("\u0000", "")
                 except UnicodeDecodeError:
-                    cleanedExif[k] = str(v).replace("\x00", "")
+                    cleanedExif[k] = str(v).replace("\x00", "").replace("\u0000", "")
             elif isinstance(v, str):
-                cleanedExif[k] = v.replace("\x00", "")
+                cleanedExif[k] = v.replace("\x00", "").replace("\u0000", "")
             else:
                 try:
                     cleanedExif[k] = str(v)
@@ -694,6 +662,7 @@ def readPictureMetadata(picture: bytes, lang: Optional[str] = "en") -> dict:
         except:
             logging.exception("Can't read EXIF tag: " + k + " " + str(type(v)))
 
+    metadata["exif"] = cleanedExif
     return metadata
 
 

geovisio/utils/semantics.py

@@ -0,0 +1,120 @@
+from dataclasses import dataclass
+from uuid import UUID
+from psycopg import Cursor
+from psycopg.sql import SQL, Identifier
+from psycopg.types.json import Jsonb
+from psycopg.errors import UniqueViolation
+from pydantic import BaseModel, ConfigDict, Field
+from typing import List
+from enum import Enum
+
+from geovisio import errors
+
+
+class TagAction(str, Enum):
+    """Actions to perform on a tag list"""
+
+    add = "add"
+    delete = "delete"
+
+
+class SemanticTagUpdate(BaseModel):
+    """Parameters used to update a tag list"""
+
+    action: TagAction = Field(default=TagAction.add)
+    """Action to perform on the tag list. The default action is `add` which will add the given tag to the list.
+    The action can also be to `delete` the key/value"""
+    key: str = Field(max_length=256)
+    """Key of the tag to update limited to 256 characters"""
+    value: str = Field(max_length=2048)
+    """Value of the tag to update limited ot 2048 characters"""
+
+    model_config = ConfigDict(use_attribute_docstrings=True)
+
+
+class SemanticTag(BaseModel):
+    key: str
+    """Key of the tag"""
+    value: str
+    """Value of the tag"""
+
+
+class EntityType(Enum):
+
+    pic = "picture_id"
+    seq = "sequence_id"
+    annotation = "annotation_id"
+
+    def entitiy_id_field(self) -> Identifier:
+        return Identifier(self.value)
+
+
+@dataclass
+class Entity:
+    type: EntityType
+    id: UUID
+
+    def get_table(self) -> Identifier:
+        match self.type:
+            case EntityType.pic:
+                return Identifier("pictures_semantics")
+            case EntityType.seq:
+                return Identifier("sequences_semantics")
+            case EntityType.annotation:
+                return Identifier("annotations_semantics")
+            case _:
+                raise ValueError(f"Unknown entity type: {self.type}")
+
+    def get_history_table(self) -> Identifier:
+        match self.type:
+            case EntityType.pic:
+                return Identifier("pictures_semantics_history")
+            case EntityType.seq:
+                return Identifier("sequences_semantics_history")
+            case EntityType.annotation:
+                return Identifier("annotations_semantics_history")
+            case _:
+                raise ValueError(f"Unknown entity type: {self.type}")
+
+
+def update_tags(cursor: Cursor, entity: Entity, actions: List[SemanticTagUpdate], account: UUID) -> SemanticTag:
+    """Update tags for an entity
+    Note: this should be done inside an autocommit transaction
+    """
+    table_name = entity.get_table()
+    fields = [entity.type.entitiy_id_field(), Identifier("key"), Identifier("value")]
+    tag_to_add = [t for t in actions if t.action == TagAction.add]
+    tag_to_delete = [t for t in actions if t.action == TagAction.delete]
+    try:
+        if tag_to_delete:
+            cursor.execute(SQL("CREATE TEMPORARY TABLE tags_to_delete(key TEXT, value TEXT) ON COMMIT DROP"))
+            with cursor.copy(SQL("COPY tags_to_delete (key, value) FROM STDIN")) as copy:
+                for tag in tag_to_delete:
+                    copy.write_row((tag.key, tag.value))
+            cursor.execute(
+                SQL(
+                    """DELETE FROM {table}
+    WHERE {entity_id} = %(entity)s 
+    AND (key, value) IN (
+        SELECT key, value FROM tags_to_delete
+    )"""
+                ).format(table=table_name, entity_id=entity.type.entitiy_id_field()),
+                {"entity": entity.id, "key_values": [(t.key, t.value) for t in tag_to_delete]},
+            )
+        if tag_to_add:
+            with cursor.copy(SQL("COPY {table} ({fields}) FROM STDIN").format(table=table_name, fields=SQL(",").join(fields))) as copy:
+                for tag in tag_to_add:
+                    copy.write_row((entity.id, tag.key, tag.value))
+        if tag_to_add or tag_to_delete:
+            # we track the history changes of the semantic tags
+            cursor.execute(
+                SQL("INSERT INTO {history_table} ({entity_id_field}, account_id, updates) VALUES (%(id)s, %(account)s, %(tags)s)").format(
+                    history_table=entity.get_history_table(), entity_id_field=entity.type.entitiy_id_field()
+                ),
+                {"id": entity.id, "account": account, "tags": Jsonb([t.model_dump() for t in tag_to_add + tag_to_delete])},
+            )
+    except UniqueViolation as e:
+        # if the tag already exists, we don't want to add it again
+        raise errors.InvalidAPIUsage(
+            "Impossible to add semantic tags because of duplicates", payload={"details": {"duplicate": e.diag.message_detail}}
+        )

geovisio/utils/sequences.py

@@ -158,9 +158,18 @@ def get_collections(request: CollectionsRequest) -> Collections:
                 s.user_agent,
                 ROUND(ST_Length(s.geom::geography)) / 1000 AS length_km,
                 s.computed_h_pixel_density,
-                s.computed_gps_accuracy
+                s.computed_gps_accuracy,
+                t.semantics
             FROM sequences s
             LEFT JOIN accounts on s.account_id = accounts.id
+            LEFT JOIN (
+                    SELECT sequence_id, json_agg(json_strip_nulls(json_build_object(
+                        'key', key,
+                        'value', value
+                    ))) AS semantics
+                    FROM sequences_semantics
+                    GROUP BY sequence_id
+                ) t ON t.sequence_id = s.id
             WHERE {filter}
             ORDER BY {order1}
             LIMIT {limit}

geovisio/utils/tokens.py

@@ -46,7 +46,7 @@ def get_account_from_jwt_token(jwt_token: str) -> auth.Account:
         # check token existence
         records = cursor.execute(
             """SELECT
-    t.account_id AS id, a.name, a.oauth_provider, a.oauth_id, a.role
+    t.account_id AS id, a.name, a.oauth_provider, a.oauth_id, a.role, a.collaborative_metadata, a.tos_accepted
 FROM tokens t
 LEFT OUTER JOIN accounts a ON t.account_id = a.id
 WHERE t.id = %(token)s""",
@@ -61,11 +61,13 @@ WHERE t.id = %(token)s""",
             )
 
         return auth.Account(
-            id=str(records["id"]),
+            id=records["id"],
             name=records["name"],
             oauth_provider=records["oauth_provider"],
             oauth_id=records["oauth_id"],
-            role=auth.AccountRole[records["role"]],
+            role=auth.AccountRole(records["role"]),
+            collaborative_metadata=records["collaborative_metadata"],
+            tos_accepted=records["tos_accepted"],
         )