geovisio 2.6.0__py3-none-any.whl → 2.7.1__py3-none-any.whl

geovisio/utils/pictures.py

@@ -1,8 +1,9 @@
 import math
 from typing import Dict, Optional
+from uuid import UUID
 from flask import current_app, redirect, send_file
+from flask_babel import gettext as _
 import os
-import psycopg
 from psycopg.rows import dict_row
 import requests
 from PIL import Image
@@ -11,9 +12,10 @@ import fs.base
 import logging
 from dataclasses import asdict
 from fs.path import dirname
-from geopic_tag_reader import reader
-from psycopg.errors import UniqueViolation
+from psycopg.errors import UniqueViolation, InvalidParameterValue
 from geovisio import utils, errors
+from geopic_tag_reader import reader
+import re
 
 log = logging.getLogger(__name__)
 
@@ -259,7 +261,7 @@ def generatePictureDerivates(fs, picture, sizing, outputFolder, type="equirectan
     return True
 
 
-def removeAllFiles(picId: str):
+def removeAllFiles(picId: UUID):
     """
     Remove all picture's associated files (the picture and all its derivate)
     """
@@ -293,10 +295,15 @@ def _remove_empty_parent_dirs(fs: fs.base.FS, dir: str):
 def checkFormatParam(format):
     """Verify that user asks for a valid image format"""
 
-    valid = ["jpg", "webp"]
+    valid = ["jpg"]
     if format not in valid:
         raise errors.InvalidAPIUsage(
-            "Invalid '" + format + "' format for image, only the following formats are available: " + ", ".join(valid), status_code=404
+            _(
+                "Invalid '%(format)s' format for image, only the following formats are available: %(allowed_formats)s",
+                format=format,
+                allowed_formats=", ".join(valid),
+            ),
+            status_code=404,
         )
 
 
@@ -308,11 +315,9 @@ def sendInFormat(picture, picFormat, httpFormat):
 
     if picFormat == httpFormat:
         return send_file(picture, mimetype="image/" + httpFormat)
-    else:
-        imgio = io.BytesIO()
-        Image.open(picture).save(imgio, format=httpFormat, quality=90)
-        imgio.seek(0)
-        return send_file(imgio, mimetype="image/" + httpFormat)
+
+    # We do not want on the fly conversions
+    raise errors.InvalidAPIUsage("Picture is not available in this format", status_code=404)
 
 
 def getPublicDerivatePictureExternalUrl(pictureId: str, format: str, derivateFileName: str) -> Optional[str]:
@@ -390,9 +395,9 @@ def checkPictureStatus(fses, pictureId):
     account = utils.auth.get_current_account()
     accountId = account.id if account is not None else None
     # Check picture availability + status
-    with psycopg.connect(current_app.config["DB_URL"], row_factory=dict_row) as db:
-        picMetadata = db.execute(
-            """
+    picMetadata = utils.db.fetchone(
+        current_app,
+        """
 			SELECT
 				p.status,
 				(p.metadata->>'cols')::int AS cols,
@@ -405,41 +410,42 @@ def checkPictureStatus(fses, pictureId):
 			JOIN sequences s ON s.id = sp.seq_id
    			WHERE p.id = %s
 		""",
-            [pictureId],
-        ).fetchone()
+        [pictureId],
+        row_factory=dict_row,
+    )
 
-        if picMetadata is None:
-            raise errors.InvalidAPIUsage("Picture can't be found, you may check its ID", status_code=404)
+    if picMetadata is None:
+        raise errors.InvalidAPIUsage(_("Picture can't be found, you may check its ID"), status_code=404)
 
-        if (picMetadata["status"] != "ready" or picMetadata["seq_status"] != "ready") and accountId != str(picMetadata["account_id"]):
-            raise errors.InvalidAPIUsage("Picture is not available (either hidden by admin or processing)", status_code=403)
+    if (picMetadata["status"] != "ready" or picMetadata["seq_status"] != "ready") and accountId != str(picMetadata["account_id"]):
+        raise errors.InvalidAPIUsage(_("Picture is not available (either hidden by admin or processing)"), status_code=403)
 
-        if current_app.config.get("PICTURE_PROCESS_DERIVATES_STRATEGY") == "PREPROCESS":
-            # if derivates are always generated, not need for other checks
-            return picMetadata
+    if current_app.config.get("PICTURE_PROCESS_DERIVATES_STRATEGY") == "PREPROCESS":
+        # if derivates are always generated, not need for other checks
+        return picMetadata
 
-        # Check original image availability
-        if not fses.permanent.exists(utils.pictures.getHDPicturePath(pictureId)):
-            raise errors.InvalidAPIUsage("HD Picture file is not available", status_code=500)
+    # Check original image availability
+    if not fses.permanent.exists(utils.pictures.getHDPicturePath(pictureId)):
+        raise errors.InvalidAPIUsage(_("HD Picture file is not available"), status_code=500)
 
-        # Check derivates availability
-        if areDerivatesAvailable(fses.derivates, pictureId, picMetadata["type"]):
-            return picMetadata
-        else:
-            picDerivates = utils.pictures.getPictureFolderPath(pictureId)
+    # Check derivates availability
+    if areDerivatesAvailable(fses.derivates, pictureId, picMetadata["type"]):
+        return picMetadata
+    else:
+        picDerivates = utils.pictures.getPictureFolderPath(pictureId)
 
-            # Try to create derivates folder if it doesn't exist yet
-            fses.derivates.makedirs(picDerivates, recreate=True)
+        # Try to create derivates folder if it doesn't exist yet
+        fses.derivates.makedirs(picDerivates, recreate=True)
 
-            picture = Image.open(fses.permanent.openbin(utils.pictures.getHDPicturePath(pictureId)))
+        picture = Image.open(fses.permanent.openbin(utils.pictures.getHDPicturePath(pictureId)))
 
-            # Force generation of derivates
-            if utils.pictures.generatePictureDerivates(
-                fses.derivates, picture, utils.pictures.getPictureSizing(picture), picDerivates, picMetadata["type"]
-            ):
-                return picMetadata
-            else:
-                raise errors.InvalidAPIUsage("Picture derivates file are not available", status_code=500)
+        # Force generation of derivates
+        if utils.pictures.generatePictureDerivates(
+            fses.derivates, picture, utils.pictures.getPictureSizing(picture), picDerivates, picMetadata["type"]
+        ):
+            return picMetadata
+        else:
+            raise errors.InvalidAPIUsage(_("Picture derivates file are not available"), status_code=500)
 
 
 def sendThumbnail(pictureId, format):
@@ -456,7 +462,7 @@ def sendThumbnail(pictureId, format):
     try:
         picture = fses.derivates.openbin(utils.pictures.getPictureFolderPath(pictureId) + "/thumb.jpg")
     except:
-        raise errors.InvalidAPIUsage("Unable to read picture on filesystem", status_code=500)
+        raise errors.InvalidAPIUsage(_("Unable to read picture on filesystem"), status_code=500)
 
     return sendInFormat(picture, "jpeg", format)
 
@@ -492,13 +498,22 @@ class PicturePositionConflict(Exception):
         super().__init__()
 
 
-class MetadataReadingError(Exception):
+class InvalidMetadataValue(Exception):
     def __init__(self, details):
         super().__init__()
         self.details = details
 
 
-def insertNewPictureInDatabase(db, sequenceId, position, pictureBytes, associatedAccountID, addtionalMetadata):
+class MetadataReadingError(Exception):
+    def __init__(self, details, missing_mandatory_tags=[]):
+        super().__init__()
+        self.details = details
+        self.missing_mandatory_tags = missing_mandatory_tags
+
+
+def insertNewPictureInDatabase(
+    db, sequenceId, position, pictureBytes, associatedAccountID, additionalMetadata, uploadSetID=None, lang="en"
+):
     """Inserts a new 'pictures' entry in the database, from a picture file.
     Database is not committed in this function, to make entry definitively stored
     you have to call db.commit() after or use an autocommit connection.
@@ -526,7 +541,7 @@ def insertNewPictureInDatabase(db, sequenceId, position, pictureBytes, associate
 
     # Create a fully-featured metadata object
     picturePillow = Image.open(io.BytesIO(pictureBytes))
-    metadata = readPictureMetadata(pictureBytes) | utils.pictures.getPictureSizing(picturePillow) | addtionalMetadata
+    metadata = readPictureMetadata(pictureBytes, lang) | utils.pictures.getPictureSizing(picturePillow) | additionalMetadata
 
     # Remove cols/rows information for flat pictures
     if metadata["type"] == "flat":
@@ -534,37 +549,48 @@ def insertNewPictureInDatabase(db, sequenceId, position, pictureBytes, associate
         metadata.pop("rows")
 
     # Create a lighter metadata field to remove duplicates fields
-    lighterMetadata = dict(filter(lambda v: v[0] not in ["ts", "heading", "lon", "lat", "exif"], metadata.items()))
+    lighterMetadata = dict(
+        filter(lambda v: v[0] not in ["ts", "heading", "lon", "lat", "exif", "originalContentMd5", "ts_by_source"], metadata.items())
+    )
     if lighterMetadata.get("tagreader_warnings") is not None and len(lighterMetadata["tagreader_warnings"]) == 0:
         del lighterMetadata["tagreader_warnings"]
     lighterMetadata["tz"] = metadata["ts"].tzname()
+    if metadata.get("ts_by_source", {}).get("gps") is not None:
+        lighterMetadata["ts_gps"] = metadata["ts_by_source"]["gps"].isoformat()
+    if metadata.get("ts_by_source", {}).get("camera") is not None:
+        lighterMetadata["ts_camera"] = metadata["ts_by_source"]["camera"].isoformat()
 
     exif = cleanupExif(metadata["exif"])
 
     with db.transaction():
         # Add picture metadata to database
-        picId = db.execute(
-            """
-			INSERT INTO pictures (ts, heading, metadata, geom, account_id, exif)
-			VALUES (%s, %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326), %s, %s)
-			RETURNING id
-		""",
-            (
-                metadata["ts"].isoformat(),
-                metadata["heading"],
-                Jsonb(lighterMetadata),
-                metadata["lon"],
-                metadata["lat"],
-                associatedAccountID,
-                Jsonb(exif),
-            ),
-        ).fetchone()[0]
+        try:
+            picId = db.execute(
+                """
+                INSERT INTO pictures (ts, heading, metadata, geom, account_id, exif, original_content_md5, upload_set_id)
+                VALUES (%s, %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326), %s, %s, %s, %s)
+                RETURNING id
+            """,
+                (
+                    metadata["ts"].isoformat(),
+                    metadata["heading"],
+                    Jsonb(lighterMetadata),
+                    metadata["lon"],
+                    metadata["lat"],
+                    associatedAccountID,
+                    Jsonb(exif),
+                    metadata.get("originalContentMd5"),
+                    uploadSetID,
+                ),
+            ).fetchone()[0]
+        except InvalidParameterValue as e:
+            raise InvalidMetadataValue(e.diag.message_primary) from e
 
         # Process field of view for each pictures
         # Flat pictures = variable fov
         if metadata["type"] == "flat":
             make, model = metadata.get("make"), metadata.get("model")
-            if make is not None and model is not None:
+            if make is not None and model is not None and metadata["focal_length"] != 0:
                 db.execute("SET pg_trgm.similarity_threshold = 0.9")
                 db.execute(
                     """
@@ -594,11 +620,11 @@ def insertNewPictureInDatabase(db, sequenceId, position, pictureBytes, associate
 			""",
                 [picId],
             )
-
-        try:
-            db.execute("INSERT INTO sequences_pictures(seq_id, rank, pic_id) VALUES(%s, %s, %s)", [sequenceId, position, picId])
-        except UniqueViolation as e:
-            raise PicturePositionConflict() from e
+        if sequenceId is not None:
+            try:
+                db.execute("INSERT INTO sequences_pictures(seq_id, rank, pic_id) VALUES(%s, %s, %s)", [sequenceId, position, picId])
+            except UniqueViolation as e:
+                raise PicturePositionConflict() from e
 
     return picId
 
@@ -606,36 +632,26 @@ def insertNewPictureInDatabase(db, sequenceId, position, pictureBytes, associate
 # Note: we don't want to store and expose exif binary fields as they are difficult to use and take a lot of storage in the database (~20% for maker notes only)
 # This list has been queried from real data (cf [this comment](https://gitlab.com/panoramax/server/api/-/merge_requests/241#note_1790580636)).
 # Update this list (and do a sql migration) if new binary fields are added
+# Note that tags ending in ".0xXXXX" are automatically striped by a regex
 BLACK_LISTED_BINARY_EXIF_FIELDS = set(
     [
         "Exif.Photo.MakerNote",
-        "Exif.Photo.0xea1c",
-        "Exif.Image.0xea1c",
         "Exif.Canon.CameraInfo",
         "Exif.Image.PrintImageMatching",
-        "Exif.Image.0xc6d3",
         "Exif.Panasonic.FaceDetInfo",
         "Exif.Panasonic.DataDump",
-        "Exif.Image.0xc6d2",
         "Exif.Canon.CustomFunctions",
         "Exif.Canon.AFInfo",
-        "Exif.Canon.0x4011",
-        "Exif.Canon.0x4019",
         "Exif.Canon.ColorData",
         "Exif.Canon.DustRemovalData",
         "Exif.Canon.VignettingCorr",
         "Exif.Canon.AFInfo3",
-        "Exif.Canon.0x001f",
-        "Exif.Canon.0x0018",
         "Exif.Canon.ContrastInfo",
-        "Exif.Canon.0x002e",
-        "Exif.Canon.0x0022",
-        "Exif.Photo.0x9aaa",
     ]
 )
 
 
-def readPictureMetadata(picture: bytes) -> dict:
+def readPictureMetadata(picture: bytes, lang: Optional[str] = "en") -> dict:
     """Extracts metadata from picture file
 
     Parameters
@@ -652,15 +668,16 @@ def readPictureMetadata(picture: bytes) -> dict:
     """
 
     try:
-        metadata = asdict(reader.readPictureMetadata(picture))
-    except Exception as e:
-        raise MetadataReadingError(details=str(e))
+        metadata = asdict(reader.readPictureMetadata(picture, lang))
+    except reader.PartialExifException as e:
+        tags = [t for t in e.missing_mandatory_tags if t not in ("lon", "lat")]
+        if "lon" in e.missing_mandatory_tags or "lat" in e.missing_mandatory_tags:
+            tags.append("location")  # lat/lon is too much detail for missing metadatas, we replace those by 'location'
+        raise MetadataReadingError(details=str(e), missing_mandatory_tags=tags)
 
     # Cleanup raw EXIF tags to avoid SQL issues
     cleanedExif = {}
-    for k, v in metadata["exif"].items():
-        if k in BLACK_LISTED_BINARY_EXIF_FIELDS:
-            continue
+    for k, v in cleanupExif(metadata["exif"]).items():
         try:
             if isinstance(v, bytes):
                 try:
@@ -680,15 +697,20 @@ def readPictureMetadata(picture: bytes) -> dict:
     return metadata
 
 
+EXIF_KEY_HEX_RGX = r"\.0x[0-9a-fA-F]+$"
+
+
 def cleanupExif(exif: Optional[Dict[str, str]]) -> Optional[Dict[str, str]]:
-    """Removes binary fields from exif
+    """Removes binary or undocumented fields from EXIF tags
     >>> cleanupExif({'A': 'B', 'Exif.Canon.AFInfo': 'Blablabla'})
     {'A': 'B'}
     >>> cleanupExif({'A': 'B', 'Exif.Photo.MakerNote': 'Blablabla'})
     {'A': 'B'}
+    >>> cleanupExif({'A': 'B', 'Exif.Sony.0x1234': 'Blablabla'})
+    {'A': 'B'}
     """
 
     if exif is None:
         return None
 
-    return {k: v for k, v in exif.items() if k not in BLACK_LISTED_BINARY_EXIF_FIELDS}
+    return {k: v for k, v in exif.items() if not re.search(EXIF_KEY_HEX_RGX, k) and k not in BLACK_LISTED_BINARY_EXIF_FIELDS}

geovisio/utils/reports.py

@@ -0,0 +1,171 @@
+from enum import Enum
+from uuid import UUID
+from typing import Optional, List
+from typing_extensions import Self
+from datetime import datetime
+from pydantic import BaseModel, ConfigDict
+from geovisio.utils import db
+from geovisio.errors import InvalidAPIUsage
+from flask import current_app
+from psycopg.sql import SQL
+from psycopg.rows import class_row
+
+
+class ReportType(Enum):
+    blur_missing = "blur_missing"
+    blur_excess = "blur_excess"
+    inappropriate = "inappropriate"
+    privacy = "privacy"
+    picture_low_quality = "picture_low_quality"
+    mislocated = "mislocated"
+    copyright = "copyright"
+    other = "other"
+
+
+class ReportStatus(Enum):
+    open = "open"
+    open_autofix = "open_autofix"
+    waiting = "waiting"
+    closed_solved = "closed_solved"
+    closed_ignored = "closed_ignored"
+
+
+class Report(BaseModel):
+    """A Report is a problem reported from a third-party about a picture or a sequence."""
+
+    id: UUID
+    issue: ReportType
+    status: ReportStatus
+    picture_id: Optional[UUID]
+    sequence_id: Optional[UUID]
+    ts_opened: datetime
+    ts_closed: Optional[datetime]
+    reporter_account_id: Optional[UUID]
+    reporter_email: Optional[str]
+    resolver_account_id: Optional[UUID]
+    reporter_comments: Optional[str]
+    resolver_comments: Optional[str]
+
+    model_config = ConfigDict(use_enum_values=True, ser_json_timedelta="float")
+
+    def for_public(self) -> Self:
+        """Report version for public display (without report email and admin comments)"""
+        return Report(
+            id=self.id,
+            issue=self.issue,
+            status=self.status,
+            picture_id=self.picture_id,
+            sequence_id=self.sequence_id,
+            ts_opened=self.ts_opened,
+            ts_closed=self.ts_closed,
+            reporter_account_id=self.reporter_account_id,
+            reporter_email=None,
+            resolver_account_id=self.resolver_account_id,
+            reporter_comments=self.reporter_comments,
+            resolver_comments=None,
+        )
+
+
+class Reports(BaseModel):
+    reports: List[Report]
+
+
+def get_report(id: UUID) -> Optional[Report]:
+    """Get the Report corresponding to the ID"""
+    db_report = db.fetchone(
+        current_app,
+        SQL("SELECT * FROM reports WHERE id = %(id)s"),
+        {"id": id},
+        row_factory=class_row(Report),
+    )
+
+    return db_report
+
+
+def is_picture_owner(report: Report, account_id: UUID):
+    """Check if given account is owner of picture concerned by report"""
+
+    isOwner = False
+    if report.picture_id is not None:
+        concernedPic = db.fetchone(
+            current_app,
+            SQL("SELECT id FROM pictures WHERE id = %(id)s AND account_id = %(uid)s"),
+            {"id": report.picture_id, "uid": account_id},
+        )
+        isOwner = concernedPic is not None
+    elif report.sequence_id is not None:
+        concernedSeq = db.fetchone(
+            current_app,
+            SQL("SELECT id FROM sequences WHERE id = %(id)s AND account_id = %(uid)s"),
+            {"id": report.sequence_id, "uid": account_id},
+        )
+        isOwner = concernedSeq is not None
+    return isOwner
+
+
+def _parse_filter(filter: Optional[str]) -> SQL:
+    """
+    Parse a filter string and return a SQL expression
+
+    >>> _parse_filter('')
+    SQL('TRUE')
+    >>> _parse_filter(None)
+    SQL('TRUE')
+    >>> _parse_filter("status = \'open\'")
+    SQL("(r.status = \'open\')")
+    >>> _parse_filter("status IN (\'open_autofix\', \'waiting\')")
+    SQL("r.status IN (\'open_autofix\', \'waiting\')")
+    >>> _parse_filter("reporter = \'me\'")
+    SQL('(reporter_account_id = %(account_id)s)')
+    >>> _parse_filter("owner = \'me\'")
+    SQL('(COALESCE(p.account_id, s.account_id) = %(account_id)s)')
+    >>> _parse_filter("status IN (\'open\', \'open_autofix\', \'waiting\') AND (owner = \'me\' OR reporter = \'me\')")
+    SQL("(r.status IN (\'open\', \'open_autofix\', \'waiting\') AND ((COALESCE(p.account_id, s.account_id) = %(account_id)s) OR (reporter_account_id = %(account_id)s)))")
+    """
+    if not filter:
+        return SQL("TRUE")
+    from pygeofilter.backends.sql import to_sql_where
+    from pygeofilter.parsers.ecql import parse as ecql_parser
+
+    try:
+        filterAst = ecql_parser(filter)
+        fieldsToFilter = {
+            "status": "r.status",
+            "reporter": "reporter_account_id",
+            "owner": "COALESCE(p.account_id, s.account_id)",
+        }
+
+        f = to_sql_where(filterAst, fieldsToFilter).replace('"', "").replace("'me'", "%(account_id)s")  # type: ignore
+        return SQL(f)  # type: ignore
+    except Exception as e:
+        print(e)
+        raise InvalidAPIUsage(_("Unsupported filter parameter"), status_code=400)
+
+
+def list_reports(account_id: UUID, limit: int = 100, filter: Optional[str] = None, forceAccount: bool = True) -> Reports:
+    filter_sql = _parse_filter(filter)
+    if forceAccount:
+        filter_sql = SQL(" ").join(
+            [SQL("(COALESCE(p.account_id, s.account_id) = %(account_id)s OR reporter_account_id = %(account_id)s) AND "), filter_sql]
+        )
+
+    l = db.fetchall(
+        current_app,
+        SQL(
+            """
+        SELECT
+            r.*,
+            COALESCE(p.account_id, s.account_id) AS owner_account_id
+        FROM reports r
+        LEFT JOIN pictures p ON r.picture_id = p.id
+        LEFT JOIN sequences s ON r.sequence_id = s.id
+        WHERE {filter}
+        ORDER BY ts_opened DESC
+        LIMIT %(limit)s
+        """
+        ).format(filter=filter_sql),
+        {"account_id": account_id, "limit": limit},
+        row_factory=class_row(Report),
+    )
+
+    return Reports(reports=l)