geovisio 2.6.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

geovisio/utils/auth.py

@@ -1,12 +1,17 @@
 import flask
 from flask import current_app, url_for, session, redirect, request
+from flask_babel import gettext as _
 from functools import wraps
 from authlib.integrations.flask_client import OAuth
 from dataclasses import dataclass
 from abc import ABC, abstractmethod
 from typing import Any
 from typing import Optional
+from enum import Enum
+from pydantic import BaseModel, ConfigDict, Field
 import sentry_sdk
+from psycopg.rows import dict_row
+from geovisio.utils import db
 
 
 ACCOUNT_KEY = "account"  # Key in flask's session with the account's information
@@ -144,12 +149,45 @@ def make_auth(app):
     return oauth
 
 
-@dataclass
-class Account(object):
+class AccountRole(Enum):
+    user = "user"
+    admin = "admin"
+
+
+class Account(BaseModel):
     id: str
     name: str
-    oauth_provider: str
-    oauth_id: str
+    oauth_provider: Optional[str] = None
+    oauth_id: Optional[str] = None
+
+    model_config = ConfigDict(extra="forbid")
+
+    def __init__(self, role: Optional[AccountRole] = None, **kwargs) -> None:
+        super().__init__(**kwargs)
+        self.role = role
+
+    # Note: this field is excluded since we do not want to persist it in the cookie. It will be fetched from the database if needed
+    # and accessed though the `role` property
+    role_: Optional[AccountRole] = Field(default=None, exclude=True)
+
+    def can_check_reports(self):
+        """Is account legitimate to read any report ?"""
+        return self.role == AccountRole.admin
+
+    def can_edit_excluded_areas(self):
+        """Is account legitimate to read and edit excluded areas ?"""
+        return self.role == AccountRole.admin
+
+    @property
+    def role(self) -> AccountRole:
+        if self.role_ is None:
+            role = db.fetchone(current_app, "SELECT role FROM accounts WHERE id = %s", (self.id,), row_factory=dict_row)
+            self.role_ = AccountRole(role["role"])
+        return self.role_
+
+    @role.setter
+    def role(self, r: AccountRole) -> None:
+        self.role_ = r
 
 
 def login_required():
@@ -160,7 +198,7 @@ def login_required():
         def decorator(*args, **kwargs):
             account = get_current_account()
             if not account:
-                return flask.abort(flask.make_response(flask.jsonify(message="Authentication is mandatory"), 401))
+                return flask.abort(flask.make_response(flask.jsonify(message=_("Authentication is mandatory")), 401))
             kwargs["account"] = account
 
             return f(*args, **kwargs)
@@ -236,7 +274,7 @@ class UnknowAccountException(Exception):
     status_code = 401
 
     def __init__(self):
-        msg = f"No account with this oauth id is know, you should login first"
+        msg = "No account with this oauth id is know, you should login first"
         super().__init__(msg)
 
 
@@ -244,7 +282,7 @@ class LoginRequiredException(Exception):
     status_code = 401
 
     def __init__(self):
-        msg = f"You should login to request this API"
+        msg = "You should login to request this API"
         super().__init__(msg)
 
 
@@ -259,9 +297,10 @@ def get_current_account():
                     Account: the current logged account, None if nobody is logged
     """
     if ACCOUNT_KEY in session:
-        session_account = Account(**session[ACCOUNT_KEY])
+        a = session[ACCOUNT_KEY]
+        session_account = Account(**a)
 
-        sentry_sdk.set_user(session_account.__dict__)
+        sentry_sdk.set_user(session_account.model_dump(exclude_none=True))
         return session_account
 
     bearer_token = _get_bearer_token()
@@ -269,7 +308,7 @@ def get_current_account():
         from geovisio.utils import tokens
 
         a = tokens.get_account_from_jwt_token(bearer_token)
-        sentry_sdk.set_user(a.__dict__)
+        sentry_sdk.set_user(a.model_dump(exclude_none=True))
         return a
 
     return None
@@ -288,5 +327,5 @@ def _get_bearer_token() -> Optional[str]:
     if not auth_header.startswith("Bearer "):
         from geovisio.utils.tokens import InvalidTokenException
 
-        raise InvalidTokenException("Only Bearer token are supported")
+        raise InvalidTokenException(_("Only Bearer token are supported"))
     return auth_header.split(" ")[1]

geovisio/utils/db.py

@@ -0,0 +1,65 @@
+from psycopg_pool import ConnectionPool
+from contextlib import contextmanager
+from typing import Optional
+
+
+def create_db_pool(app):
+    """
+    Create Database connection pool
+
+    Note: all returned connections are autocommit connection. If it's not the wanted behavior, wrap the query in an explicit transaction, or acquire a connection outside of the pool.
+    """
+    if hasattr(app, "pool"):
+        return
+    min_size = int(app.config["DB_MIN_CNX"])
+    max_size = int(app.config["DB_MAX_CNX"])
+    statement_timeout = app.config["DB_STATEMENT_TIMEOUT"]
+    args = {"autocommit": True}
+    if statement_timeout > 0:
+        args["options"] = f"-c statement_timeout={statement_timeout}"
+    app.pool = ConnectionPool(conninfo=app.config["DB_URL"], min_size=min_size, max_size=max_size, open=True, kwargs=args)
+    # add also a connection pool without timeout for queries that are known to be long
+    # This is useful for example for refreshing the pictures_grid materialized view
+    app.long_queries_pool = ConnectionPool(
+        conninfo=app.config["DB_URL"], min_size=0, max_size=max_size, open=True, kwargs={"autocommit": True}
+    )
+
+
+@contextmanager
+def conn(app, timeout: Optional[float] = None):
+    """Get a psycopg connection from the connection pool"""
+    with app.pool.connection(timeout=timeout) as conn:
+        yield conn
+
+
+@contextmanager
+def cursor(app, timeout: Optional[float] = None, **kwargs):
+    """Get a psycopg cursor from the connection pool"""
+    with app.pool.connection(timeout=timeout) as conn:
+        yield conn.cursor(**kwargs)
+
+
+@contextmanager
+def execute(app, sql, params=None, timeout: Optional[float] = None, **kwargs):
+    """Simple helpers to simplify simple calls to get a cursor and execute a query on it"""
+    with cursor(app, timeout=timeout, **kwargs) as c:
+        yield c.execute(sql, params=params)
+
+
+def fetchone(app, sql, params=None, timeout: Optional[float] = None, **kwargs):
+    """Simple helpers to simplify simple calls to fetchone"""
+    with execute(app, sql, params, timeout=timeout, **kwargs) as q:
+        return q.fetchone()
+
+
+def fetchall(app, sql, params=None, timeout: Optional[float] = None, **kwargs):
+    """Simple helpers to simplify simple calls to fetchall"""
+    with execute(app, sql, params, timeout=timeout, **kwargs) as q:
+        return q.fetchall()
+
+
+@contextmanager
+def long_queries_conn(app, connection_timeout: Optional[float] = None):
+    """Get a psycopg connection for queries that are known to be long from the connection pool"""
+    with app.long_queries_pool.connection(timeout=connection_timeout) as conn:
+        yield conn

geovisio/utils/excluded_areas.py

@@ -0,0 +1,83 @@
+from uuid import UUID
+from typing import Optional
+from pydantic import BaseModel, ConfigDict
+from geojson_pydantic import MultiPolygon, FeatureCollection, Feature
+from geovisio.utils import db
+from geovisio.errors import InvalidAPIUsage
+from flask import current_app
+from flask_babel import gettext as _
+from psycopg.sql import SQL, Literal
+from psycopg.rows import class_row
+
+
+class ExcludedArea(BaseModel):
+    """An excluded area is a geographical boundary where pictures should not be accepted."""
+
+    id: UUID
+    label: Optional[str] = None
+    is_public: bool = False
+    account_id: Optional[UUID] = None
+
+    model_config = ConfigDict()
+
+
+ExcludedAreaFeature = Feature[MultiPolygon, ExcludedArea]
+ExcludedAreaFeatureCollection = FeatureCollection[ExcludedAreaFeature]
+
+
+def get_excluded_area(id: UUID) -> Optional[ExcludedAreaFeature]:
+    """Get the excluded area corresponding to the ID"""
+    return db.fetchone(
+        current_app,
+        SQL(
+            """SELECT id, label, is_public, account_id, ST_AsGeoJSON(geom) AS geometry
+FROM excluded_area
+WHERE id = %(id)s"""
+        ),
+        {"id": id},
+        row_factory=class_row(ExcludedAreaFeature),
+    )
+
+
+def list_excluded_areas(is_public: Optional[bool] = None, account_id: Optional[UUID] = None) -> ExcludedAreaFeatureCollection:
+    where = [Literal(True)]
+    if is_public is not None:
+        where.append(SQL("is_public IS {}").format(Literal(is_public)))
+    if account_id:
+        where.append(SQL("account_id = {}").format(Literal(account_id)))
+
+    areas = db.fetchall(
+        current_app,
+        SQL(
+            """SELECT
+    'Feature' as type,
+    json_build_object(
+        'id', id,
+        'label', label,
+        'is_public', is_public,
+        'account_id', account_id
+    ) as properties,
+    ST_AsGeoJSON(geom)::json as geometry
+FROM excluded_areas
+WHERE {}"""
+        ).format(SQL(" AND ").join(where)),
+        row_factory=class_row(ExcludedAreaFeature),
+    )
+
+    return ExcludedAreaFeatureCollection(type="FeatureCollection", features=areas)
+
+
+def delete_excluded_area(areaId: UUID, accountId: Optional[UUID] = None):
+    where = [SQL("id = {}").format(Literal(areaId))]
+    if accountId is not None:
+        where.append(SQL("account_id = {}").format(accountId))
+
+    with db.execute(
+        current_app,
+        SQL("DELETE FROM excluded_areas WHERE {}").format(SQL(" AND ").join(where)),
+    ) as res:
+        area_deleted = res.rowcount
+
+        if not area_deleted:
+            raise InvalidAPIUsage(_("Impossible to find excluded area"), status_code=404)
+        return "", 204

geovisio/utils/extent.py

@@ -0,0 +1,30 @@
+from typing import List, Optional
+from datetime import datetime
+from pydantic import BaseModel
+
+
+class Temporal(BaseModel):
+    """Temporal extent"""
+
+    interval: List[List[datetime]]
+    """Interval"""
+
+
+class Spatial(BaseModel):
+    """Spatial extent"""
+
+    bbox: List[List[float]]
+    """Bounding box"""
+
+
+class Extent(BaseModel):
+    """Spatio-temporal extents"""
+
+    temporal: Optional[Temporal]
+    spatial: Optional[Spatial]
+
+
+class TemporalExtent(BaseModel):
+    """Temporal extents (without spatial extent)"""
+
+    temporal: Optional[Temporal]

geovisio/utils/fields.py

@@ -1,6 +1,6 @@
 from enum import Enum
 from dataclasses import dataclass, field
-from typing import Any, List, Dict, Optional, Generic, TypeVar, Protocol
+from typing import Any, List, Generic, TypeVar, Protocol
 from psycopg import sql
 
 

geovisio/utils/filesystems.py

@@ -1,6 +1,5 @@
 from dataclasses import dataclass
 import fs.base
-import logging
 from fs import open_fs
 from fs.errors import ResourceNotFound
 from fs_s3fs import S3FS

geovisio/utils/link.py

@@ -0,0 +1,14 @@
+from pydantic import BaseModel
+from typing import Optional
+from flask import url_for
+
+
+class Link(BaseModel):
+    rel: str
+    type: str
+    title: Optional[str]
+    href: str
+
+
+def make_link(rel: str, route: str, title: Optional[str] = None, type: str = "application/json", **args):
+    return Link(rel=rel, type=type, title=title, href=url_for(route, **args, _external=True))

geovisio/utils/params.py

@@ -0,0 +1,20 @@
+from pydantic import ValidationError
+
+
+def validation_error(e: ValidationError):
+    """Transform a pydantic error to user friendly error, meant to be used as `payload` of a geovisio.error"""
+
+    details = []
+    for d in e.errors():
+        detail = {
+            "fields": d["loc"],
+            "error": d["msg"],
+        }
+        if d["input"]:
+            detail["input"] = d["input"]
+            if "user_agent" in detail["input"]:
+                del detail["input"]["user_agent"]
+            if len(detail["input"]) == 0:
+                del detail["input"]
+        details.append(detail)
+    return {"details": details}

geovisio/utils/pictures.py

@@ -1,8 +1,9 @@
 import math
 from typing import Dict, Optional
+from uuid import UUID
 from flask import current_app, redirect, send_file
+from flask_babel import gettext as _
 import os
-import psycopg
 from psycopg.rows import dict_row
 import requests
 from PIL import Image
@@ -11,9 +12,9 @@ import fs.base
 import logging
 from dataclasses import asdict
 from fs.path import dirname
-from geopic_tag_reader import reader
-from psycopg.errors import UniqueViolation
+from psycopg.errors import UniqueViolation, InvalidParameterValue
 from geovisio import utils, errors
+from geopic_tag_reader import reader
 
 log = logging.getLogger(__name__)
 
@@ -259,7 +260,7 @@ def generatePictureDerivates(fs, picture, sizing, outputFolder, type="equirectan
     return True
 
 
-def removeAllFiles(picId: str):
+def removeAllFiles(picId: UUID):
     """
     Remove all picture's associated files (the picture and all its derivate)
     """
@@ -293,10 +294,15 @@ def _remove_empty_parent_dirs(fs: fs.base.FS, dir: str):
 def checkFormatParam(format):
     """Verify that user asks for a valid image format"""
 
-    valid = ["jpg", "webp"]
+    valid = ["jpg"]
     if format not in valid:
         raise errors.InvalidAPIUsage(
-            "Invalid '" + format + "' format for image, only the following formats are available: " + ", ".join(valid), status_code=404
+            _(
+                "Invalid '%(format)s' format for image, only the following formats are available: %(allowed_formats)s",
+                format=format,
+                allowed_formats=", ".join(valid),
+            ),
+            status_code=404,
         )
 
 
@@ -308,11 +314,9 @@ def sendInFormat(picture, picFormat, httpFormat):
 
     if picFormat == httpFormat:
         return send_file(picture, mimetype="image/" + httpFormat)
-    else:
-        imgio = io.BytesIO()
-        Image.open(picture).save(imgio, format=httpFormat, quality=90)
-        imgio.seek(0)
-        return send_file(imgio, mimetype="image/" + httpFormat)
+
+    # We do not want on the fly conversions
+    raise errors.InvalidAPIUsage("Picture is not available in this format", status_code=404)
 
 
 def getPublicDerivatePictureExternalUrl(pictureId: str, format: str, derivateFileName: str) -> Optional[str]:
@@ -390,9 +394,9 @@ def checkPictureStatus(fses, pictureId):
     account = utils.auth.get_current_account()
     accountId = account.id if account is not None else None
     # Check picture availability + status
-    with psycopg.connect(current_app.config["DB_URL"], row_factory=dict_row) as db:
-        picMetadata = db.execute(
-            """
+    picMetadata = utils.db.fetchone(
+        current_app,
+        """
 			SELECT
 				p.status,
 				(p.metadata->>'cols')::int AS cols,
@@ -405,41 +409,42 @@ def checkPictureStatus(fses, pictureId):
 			JOIN sequences s ON s.id = sp.seq_id
    			WHERE p.id = %s
 		""",
-            [pictureId],
-        ).fetchone()
+        [pictureId],
+        row_factory=dict_row,
+    )
 
-        if picMetadata is None:
-            raise errors.InvalidAPIUsage("Picture can't be found, you may check its ID", status_code=404)
+    if picMetadata is None:
+        raise errors.InvalidAPIUsage(_("Picture can't be found, you may check its ID"), status_code=404)
 
-        if (picMetadata["status"] != "ready" or picMetadata["seq_status"] != "ready") and accountId != str(picMetadata["account_id"]):
-            raise errors.InvalidAPIUsage("Picture is not available (either hidden by admin or processing)", status_code=403)
+    if (picMetadata["status"] != "ready" or picMetadata["seq_status"] != "ready") and accountId != str(picMetadata["account_id"]):
+        raise errors.InvalidAPIUsage(_("Picture is not available (either hidden by admin or processing)"), status_code=403)
 
-        if current_app.config.get("PICTURE_PROCESS_DERIVATES_STRATEGY") == "PREPROCESS":
-            # if derivates are always generated, not need for other checks
-            return picMetadata
+    if current_app.config.get("PICTURE_PROCESS_DERIVATES_STRATEGY") == "PREPROCESS":
+        # if derivates are always generated, not need for other checks
+        return picMetadata
 
-        # Check original image availability
-        if not fses.permanent.exists(utils.pictures.getHDPicturePath(pictureId)):
-            raise errors.InvalidAPIUsage("HD Picture file is not available", status_code=500)
+    # Check original image availability
+    if not fses.permanent.exists(utils.pictures.getHDPicturePath(pictureId)):
+        raise errors.InvalidAPIUsage(_("HD Picture file is not available"), status_code=500)
 
-        # Check derivates availability
-        if areDerivatesAvailable(fses.derivates, pictureId, picMetadata["type"]):
-            return picMetadata
-        else:
-            picDerivates = utils.pictures.getPictureFolderPath(pictureId)
+    # Check derivates availability
+    if areDerivatesAvailable(fses.derivates, pictureId, picMetadata["type"]):
+        return picMetadata
+    else:
+        picDerivates = utils.pictures.getPictureFolderPath(pictureId)
 
-            # Try to create derivates folder if it doesn't exist yet
-            fses.derivates.makedirs(picDerivates, recreate=True)
+        # Try to create derivates folder if it doesn't exist yet
+        fses.derivates.makedirs(picDerivates, recreate=True)
 
-            picture = Image.open(fses.permanent.openbin(utils.pictures.getHDPicturePath(pictureId)))
+        picture = Image.open(fses.permanent.openbin(utils.pictures.getHDPicturePath(pictureId)))
 
-            # Force generation of derivates
-            if utils.pictures.generatePictureDerivates(
-                fses.derivates, picture, utils.pictures.getPictureSizing(picture), picDerivates, picMetadata["type"]
-            ):
-                return picMetadata
-            else:
-                raise errors.InvalidAPIUsage("Picture derivates file are not available", status_code=500)
+        # Force generation of derivates
+        if utils.pictures.generatePictureDerivates(
+            fses.derivates, picture, utils.pictures.getPictureSizing(picture), picDerivates, picMetadata["type"]
+        ):
+            return picMetadata
+        else:
+            raise errors.InvalidAPIUsage(_("Picture derivates file are not available"), status_code=500)
 
 
 def sendThumbnail(pictureId, format):
@@ -456,7 +461,7 @@ def sendThumbnail(pictureId, format):
     try:
         picture = fses.derivates.openbin(utils.pictures.getPictureFolderPath(pictureId) + "/thumb.jpg")
     except:
-        raise errors.InvalidAPIUsage("Unable to read picture on filesystem", status_code=500)
+        raise errors.InvalidAPIUsage(_("Unable to read picture on filesystem"), status_code=500)
 
     return sendInFormat(picture, "jpeg", format)
 
@@ -492,13 +497,21 @@ class PicturePositionConflict(Exception):
         super().__init__()
 
 
+class InvalidMetadataValue(Exception):
+    def __init__(self, details):
+        super().__init__()
+        self.details = details
+
+
 class MetadataReadingError(Exception):
     def __init__(self, details):
         super().__init__()
         self.details = details
 
 
-def insertNewPictureInDatabase(db, sequenceId, position, pictureBytes, associatedAccountID, addtionalMetadata):
+def insertNewPictureInDatabase(
+    db, sequenceId, position, pictureBytes, associatedAccountID, additionalMetadata, uploadSetID=None, lang="en"
+):
     """Inserts a new 'pictures' entry in the database, from a picture file.
     Database is not committed in this function, to make entry definitively stored
     you have to call db.commit() after or use an autocommit connection.
@@ -526,7 +539,7 @@ def insertNewPictureInDatabase(db, sequenceId, position, pictureBytes, associate
 
     # Create a fully-featured metadata object
     picturePillow = Image.open(io.BytesIO(pictureBytes))
-    metadata = readPictureMetadata(pictureBytes) | utils.pictures.getPictureSizing(picturePillow) | addtionalMetadata
+    metadata = readPictureMetadata(pictureBytes, lang) | utils.pictures.getPictureSizing(picturePillow) | additionalMetadata
 
     # Remove cols/rows information for flat pictures
     if metadata["type"] == "flat":
@@ -534,31 +547,42 @@ def insertNewPictureInDatabase(db, sequenceId, position, pictureBytes, associate
         metadata.pop("rows")
 
     # Create a lighter metadata field to remove duplicates fields
-    lighterMetadata = dict(filter(lambda v: v[0] not in ["ts", "heading", "lon", "lat", "exif"], metadata.items()))
+    lighterMetadata = dict(
+        filter(lambda v: v[0] not in ["ts", "heading", "lon", "lat", "exif", "originalContentMd5", "ts_by_source"], metadata.items())
+    )
     if lighterMetadata.get("tagreader_warnings") is not None and len(lighterMetadata["tagreader_warnings"]) == 0:
         del lighterMetadata["tagreader_warnings"]
     lighterMetadata["tz"] = metadata["ts"].tzname()
+    if metadata.get("ts_by_source", {}).get("gps") is not None:
+        lighterMetadata["ts_gps"] = metadata["ts_by_source"]["gps"].isoformat()
+    if metadata.get("ts_by_source", {}).get("camera") is not None:
+        lighterMetadata["ts_camera"] = metadata["ts_by_source"]["camera"].isoformat()
 
     exif = cleanupExif(metadata["exif"])
 
     with db.transaction():
         # Add picture metadata to database
-        picId = db.execute(
-            """
-			INSERT INTO pictures (ts, heading, metadata, geom, account_id, exif)
-			VALUES (%s, %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326), %s, %s)
-			RETURNING id
-		""",
-            (
-                metadata["ts"].isoformat(),
-                metadata["heading"],
-                Jsonb(lighterMetadata),
-                metadata["lon"],
-                metadata["lat"],
-                associatedAccountID,
-                Jsonb(exif),
-            ),
-        ).fetchone()[0]
+        try:
+            picId = db.execute(
+                """
+                INSERT INTO pictures (ts, heading, metadata, geom, account_id, exif, original_content_md5, upload_set_id)
+                VALUES (%s, %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326), %s, %s, %s, %s)
+                RETURNING id
+            """,
+                (
+                    metadata["ts"].isoformat(),
+                    metadata["heading"],
+                    Jsonb(lighterMetadata),
+                    metadata["lon"],
+                    metadata["lat"],
+                    associatedAccountID,
+                    Jsonb(exif),
+                    metadata.get("originalContentMd5"),
+                    uploadSetID,
+                ),
+            ).fetchone()[0]
+        except InvalidParameterValue as e:
+            raise InvalidMetadataValue(e.diag.message_primary) from e
 
         # Process field of view for each pictures
         # Flat pictures = variable fov
@@ -594,11 +618,11 @@ def insertNewPictureInDatabase(db, sequenceId, position, pictureBytes, associate
 			""",
                 [picId],
             )
-
-        try:
-            db.execute("INSERT INTO sequences_pictures(seq_id, rank, pic_id) VALUES(%s, %s, %s)", [sequenceId, position, picId])
-        except UniqueViolation as e:
-            raise PicturePositionConflict() from e
+        if sequenceId is not None:
+            try:
+                db.execute("INSERT INTO sequences_pictures(seq_id, rank, pic_id) VALUES(%s, %s, %s)", [sequenceId, position, picId])
+            except UniqueViolation as e:
+                raise PicturePositionConflict() from e
 
     return picId
 
@@ -635,7 +659,7 @@ BLACK_LISTED_BINARY_EXIF_FIELDS = set(
 )
 
 
-def readPictureMetadata(picture: bytes) -> dict:
+def readPictureMetadata(picture: bytes, lang: Optional[str] = "en") -> dict:
     """Extracts metadata from picture file
 
     Parameters
@@ -652,7 +676,7 @@ def readPictureMetadata(picture: bytes) -> dict:
     """
 
     try:
-        metadata = asdict(reader.readPictureMetadata(picture))
+        metadata = asdict(reader.readPictureMetadata(picture, lang))
     except Exception as e:
         raise MetadataReadingError(details=str(e))