geovisio 2.7.0__py3-none-any.whl → 2.8.0__py3-none-any.whl

geovisio/utils/upload_set.py

@@ -126,16 +126,35 @@ class FileRejectionStatus(Enum):
     """other_error means there was an error that is not related to the picture itself"""
 
 
+class FileRejectionDetails(BaseModel):
+
+    missing_fields: List[str]
+    """Mandatory metadata missing from the file. Metadata can be `datetime` or `location`."""
+
+
+class FileRejection(BaseModel):
+    """Details about a file rejection"""
+
+    reason: str
+    severity: FileRejectionStatusSeverity
+    message: Optional[str]
+    details: Optional[FileRejectionDetails]
+
+    model_config = ConfigDict(use_enum_values=True, use_attribute_docstrings=True)
+
+
 class UploadSetFile(BaseModel):
     """File uploaded in an UploadSet"""
 
     picture_id: Optional[UUID] = None
+    """ID of the picture this file belongs to. Can only be seen by the owner of the File"""
     file_name: str
     content_md5: Optional[UUID] = None
     inserted_at: datetime
     upload_set_id: UUID = Field(..., exclude=True)
     rejection_status: Optional[FileRejectionStatus] = Field(None, exclude=True)
     rejection_message: Optional[str] = Field(None, exclude=True)
+    rejection_details: Optional[Dict[str, Any]] = Field(None, exclude=True)
     file_type: Optional[FileType] = None
     size: Optional[int] = None
 
@@ -148,7 +167,7 @@ class UploadSetFile(BaseModel):
 
     @computed_field
     @property
-    def rejected(self) -> Optional[Dict[str, Any]]:
+    def rejected(self) -> Optional[FileRejection]:
         if self.rejection_status is None:
             return None
         msg = None
@@ -168,11 +187,7 @@ class UploadSetFile(BaseModel):
                 severity = FileRejectionStatusSeverity.error
         else:
             msg = self.rejection_message
-        return {
-            "reason": self.rejection_status,
-            "severity": severity,
-            "message": msg,
-        }
+        return FileRejection(reason=self.rejection_status, severity=severity, message=msg, details=self.rejection_details)
 
     @field_serializer("content_md5")
     def serialize_md5(self, md5: UUID, _info):
@@ -199,7 +214,7 @@ def get_simple_upload_set(id: UUID) -> Optional[UploadSet]:
     """Get the DB representation of an UploadSet, without associated collections and statuses"""
     u = db.fetchone(
         current_app,
-        SQL("SELECT * FROM upload_sets WHERE id = %(id)s AND not deleted"),
+        SQL("SELECT * FROM upload_sets WHERE id = %(id)s"),
         {"id": id},
         row_factory=class_row(UploadSet),
     )
@@ -310,7 +325,7 @@ SELECT u.*,
     ) AS associated_collections
 FROM upload_sets u
 LEFT JOIN upload_set_statuses us on us.upload_set_id = u.id
-WHERE u.id = %(id)s AND not deleted"""
+WHERE u.id = %(id)s"""
         ),
         {"id": id},
         row_factory=class_row(UploadSet),
@@ -384,7 +399,7 @@ def list_upload_sets(account_id: UUID, limit: int = 100, filter: Optional[str] =
                 WHERE p.upload_set_id = u.id
             ) AS nb_items
         FROM upload_sets u
-        WHERE account_id = %(account_id)s AND not deleted AND {filter}
+        WHERE account_id = %(account_id)s AND {filter}
         ORDER BY created_at ASC
         LIMIT %(limit)s
         """
@@ -434,16 +449,25 @@ def dispatch(upload_set_id: UUID):
     p.heading as heading,
     p.metadata->>'originalFileName' as file_name,
     p.metadata,
-    s.id as sequence_id
+    s.id as sequence_id,
+    f is null as has_no_file
 FROM pictures p
 LEFT JOIN sequences_pictures sp ON sp.pic_id = p.id
 LEFT JOIN sequences s ON s.id = sp.seq_id
+LEFT JOIN files f ON f.picture_id = p.id
 WHERE p.upload_set_id = %(upload_set_id)s"""
                 ),
                 {"upload_set_id": upload_set_id},
             ).fetchall()
 
+            # there is currently a bug where 2 pictures can be uploaded for the same file, so only 1 is associated to it.
+            # we want to delete one of them
+            # Those duplicates happen when a client send an upload that timeouts, but the client retries the upload and the server is not aware of this timeout (the connection is not closed).
+            # Note: later, if we are confident the bug has been removed, we might clean this code.
+            pics_to_delete_bug = [p["id"] for p in db_pics if p["has_no_file"]]
+            db_pics = [p for p in db_pics if p["has_no_file"] is False]  # pictures without files will be deleted, we don't need them
             pics_by_filename = {p["file_name"]: p for p in db_pics}
+
             pics = [
                 geopic_sequence.Picture(
                     p["file_name"],
@@ -473,9 +497,12 @@ WHERE p.upload_set_id = %(upload_set_id)s"""
             )
             reused_sequence = set()
 
-            pics_to_delete = [pics_by_filename[p.filename]["id"] for p in report.duplicate_pictures or []]
+            pics_to_delete_duplicates = [pics_by_filename[p.filename]["id"] for p in report.duplicate_pictures or []]
+            pics_to_delete = pics_to_delete_duplicates + pics_to_delete_bug
             if pics_to_delete:
-                logging.debug(f"For uploadset '{upload_set_id}', nb duplicate pictures {len(pics_to_delete)}")
+                logging.debug(
+                    f"For uploadset '{upload_set_id}', nb duplicate pictures {len(pics_to_delete_duplicates)} {f' and {len(pics_to_delete_bug)} pictures without files' if pics_to_delete_bug else ''}"
+                )
                 logging.debug(
                     f"For uploadset '{upload_set_id}', duplicate pictures {[p.filename for p in report.duplicate_pictures or []]}"
                 )
@@ -490,18 +517,11 @@ WHERE p.upload_set_id = %(upload_set_id)s"""
                         "UPDATE files SET rejection_status = 'capture_duplicate' WHERE picture_id IN (select picture_id from tmp_duplicates)"
                     )
                 )
-                cursor.execute(
-                    SQL(
-                        """INSERT INTO job_queue (picture_id, task)
-SELECT picture_id, 'delete'
-FROM tmp_duplicates
-ON CONFLICT(picture_id) DO UPDATE SET task = 'delete'"""
-                    )
-                )
+                # delete all pictures (the DB triggers will also add background jobs to delete the associated files)
+                cursor.execute(SQL("DELETE FROM pictures WHERE id IN (select picture_id FROM tmp_duplicates)"))
 
-                # ask for deletion of the pictures
-
-            for s in report.sequences:
+            number_title = len(report.sequences) > 1
+            for i, s in enumerate(report.sequences, start=1):
                 existing_sequence = next(
                     (seq for p in s.pictures if (seq := pics_by_filename[p.filename]["sequence_id"]) not in reused_sequence),
                     None,
@@ -519,6 +539,7 @@ ON CONFLICT(picture_id) DO UPDATE SET task = 'delete'"""
                     )
                     reused_sequence.add(seq_id)
                 else:
+                    new_title = f"{db_upload_set.title}{f'-{i}' if number_title else ''}"
                     seq_id = cursor.execute(
                         SQL(
                             """INSERT INTO sequences(account_id, metadata, user_agent)
@@ -527,7 +548,7 @@ RETURNING id"""
                         ),
                         {
                             "account_id": db_upload_set.account_id,
-                            "metadata": Jsonb({"title": db_upload_set.title}),
+                            "metadata": Jsonb({"title": new_title}),
                             "user_agent": db_upload_set.user_agent,
                         },
                     ).fetchone()
@@ -557,23 +578,43 @@ def insertFileInDatabase(
     picture_id: Optional[UUID] = None,
     rejection_status: Optional[FileRejectionStatus] = None,
     rejection_message: Optional[str] = None,
+    rejection_details: Optional[Dict[str, Any]] = None,
 ) -> UploadSetFile:
     """Insert a file linked to an UploadSet into the database"""
 
+    # we check if there is already a file with this name in the upload set with an associated picture.
+    # If there is no picture (because the picture has been rejected), we accept that the file is overridden
+    existing_file = cursor.execute(
+        SQL(
+            """SELECT picture_id, rejection_status
+            FROM files
+            WHERE upload_set_id = %(upload_set_id)s AND file_name = %(file_name)s AND picture_id IS NOT NULL"""
+        ),
+        params={
+            "upload_set_id": upload_set_id,
+            "file_name": file_name,
+        },
+    ).fetchone()
+    if existing_file:
+        raise errors.InvalidAPIUsage(
+            _("A different picture with the same name has already been added to this uploadset"),
+            status_code=409,
+            payload={"existing_item": {"id": existing_file["picture_id"]}},
+        )
+
     f = cursor.execute(
         SQL(
-            """
-    INSERT INTO files(
-        upload_set_id, picture_id, file_type, file_name,
-        size, content_md5, rejection_status, rejection_message)
-    VALUES (
-        %(upload_set_id)s, %(picture_id)s, %(type)s, %(file_name)s,
-        %(size)s, %(content_md5)s, %(rejection_status)s, %(rejection_message)s)
-    ON CONFLICT (upload_set_id, file_name)
-    DO UPDATE SET picture_id = %(picture_id)s, size = %(size)s, content_md5 = %(content_md5)s,
-        rejection_status = %(rejection_status)s, rejection_message = %(rejection_message)s
-    RETURNING *
-    """
+            """INSERT INTO files(
+    upload_set_id, picture_id, file_type, file_name,
+    size, content_md5, rejection_status, rejection_message, rejection_details)
+VALUES (
+    %(upload_set_id)s, %(picture_id)s, %(type)s, %(file_name)s,
+    %(size)s, %(content_md5)s, %(rejection_status)s, %(rejection_message)s, %(rejection_details)s)
+ON CONFLICT (upload_set_id, file_name)
+DO UPDATE SET picture_id = %(picture_id)s, size = %(size)s, content_md5 = %(content_md5)s,
+    rejection_status = %(rejection_status)s, rejection_message = %(rejection_message)s, rejection_details = %(rejection_details)s
+WHERE files.picture_id IS NULL -- check again that we do not override an existing picture
+RETURNING *"""
         ),
         params={
             "upload_set_id": upload_set_id,
@@ -584,6 +625,7 @@ def insertFileInDatabase(
             "content_md5": content_md5,
             "rejection_status": rejection_status,
             "rejection_message": rejection_message,
+            "rejection_details": Jsonb(rejection_details),
         },
     )
     return UploadSetFile(**f.fetchone())
@@ -602,6 +644,7 @@ def get_upload_set_files(upload_set_id: UUID) -> UploadSetFiles:
     content_md5, 
     rejection_status,
     rejection_message,
+    rejection_details,
     picture_id, 
     inserted_at
 FROM files
@@ -618,37 +661,17 @@ def delete(upload_set: UploadSet):
     """Delete an UploadSet"""
     logging.info(f"Asking for deletion of uploadset {upload_set.id}")
     with db.conn(current_app) as conn:
+        # clean job queue, to ensure no async runner are currently processing pictures/sequences/upload_sets
+        # Done outside the real deletion transaction to not trigger deadlock
+        conn.execute(SQL("DELETE FROM job_queue WHERE picture_id IN (SELECT id FROM pictures where upload_set_id = %s)"), [upload_set.id])
+        for c in upload_set.associated_collections:
+            conn.execute(SQL("DELETE FROM job_queue WHERE sequence_id = %s"), [c.id])
+
         with conn.transaction(), conn.cursor() as cursor:
             for c in upload_set.associated_collections:
                 # Mark all collections as deleted, but do not delete them
                 # Note: we do not use utils.sequences.delete_collection here, since we also want to remove the pictures not associated to any collection
-                cursor.execute(SQL("DELETE FROM job_queue WHERE sequence_id = %s"), [c.id])
                 cursor.execute(SQL("UPDATE sequences SET status = 'deleted' WHERE id = %s"), [c.id])
 
-            # check utils.sequences.delete_collection to see why picture removal is done in 2 steps
-            cursor.execute(
-                """
-INSERT INTO job_queue(picture_id, task)
-    SELECT id, 'delete'
-    FROM pictures
-    WHERE upload_set_id = %(upload_set_id)s
-ON CONFLICT (picture_id) DO UPDATE SET task = 'delete'""",
-                {"upload_set_id": upload_set.id},
-            )
-
-            # after the task have been added to the queue, we mark all picture for deletion
-            cursor.execute(
-                SQL(
-                    "UPDATE pictures SET status = 'waiting-for-delete' WHERE id IN (SELECT id FROM pictures WHERE upload_set_id = %(upload_set_id)s)"
-                ),
-                {"upload_set_id": upload_set.id},
-            )
-            # we insert the upload set deletion task in the queue after the rest, it will be done once all pictures are deleted
-            cursor.execute(
-                """INSERT INTO job_queue(upload_set_id, task) VALUES (%(upload_set_id)s, 'delete') 
-ON CONFLICT (upload_set_id) DO UPDATE SET task = 'delete'""",
-                {"upload_set_id": upload_set.id},
-            )
-
-            # and we mark it as deleted so it will disapear from the responses even if all the pictures are not yet deleted
-            cursor.execute("UPDATE upload_sets SET deleted = true WHERE id = %(upload_set_id)s", {"upload_set_id": upload_set.id})
+            # after the task have been added to the queue, we delete the upload set, and this will delete all pictures associated to it
+            cursor.execute(SQL("DELETE FROM upload_sets WHERE id = %(upload_set_id)s"), {"upload_set_id": upload_set.id})

geovisio/utils/website.py

@@ -0,0 +1,50 @@
+from typing import Optional, Dict
+
+from flask import url_for
+
+from geovisio import web
+
+WEBSITE_UNDER_SAME_HOST = "same-host"
+
+TOKEN_ACCEPTED_PAGE = "token-accepted"
+TOS_VALIDATION_PAGE = "tos-validation"
+
+
+class Website:
+    """Website associated to the API.
+    This wrapper will define the routes we expect from the website.
+
+    We should limit the interraction from the api to the website, but for some flow (especially auth flows), it's can be useful to redirect to website's page
+
+    If the url is:
+    * set to `false`, there is no associated website
+    * set to `same-host`, the website is assumed to be on the same host as the API (and will respect the host of the current request)
+    * else it should be a valid url
+    """
+
+    def __init__(self, website_url: str):
+        if website_url == WEBSITE_UNDER_SAME_HOST:
+            self.url = WEBSITE_UNDER_SAME_HOST
+        elif website_url == "false":
+            self.url = None
+        elif website_url.startswith("http"):
+            self.url = website_url
+            if not self.url.endswith("/"):
+                self.url += "/"
+        else:
+            raise Exception(
+                "API_WEBSITE_URL should either be `same-host` (and the website will be assumed to be on the same host), set to `false` if there is no website, or a valid URL"
+            )
+
+    def _to_url(self, route: str, params: Optional[Dict[str, str]] = None):
+        base_url = self.url if self.url != WEBSITE_UNDER_SAME_HOST else url_for("index", _external=True)
+
+        from urllib.parse import urlencode
+
+        return f"{base_url}{route}{f'?{urlencode(params)}' if params else ''}"
+
+    def tos_validation_page(self, params: Optional[Dict[str, str]] = None):
+        return self._to_url(TOS_VALIDATION_PAGE, params)
+
+    def cli_token_accepted_page(self, params: Optional[Dict[str, str]] = None):
+        return self._to_url(TOKEN_ACCEPTED_PAGE, params)

geovisio/web/annotations.py

@@ -0,0 +1,17 @@
+from geovisio.utils import auth
+from psycopg.rows import dict_row, class_row
+from psycopg.sql import SQL
+from geovisio.utils.semantics import Entity, EntityType, SemanticTagUpdate, update_tags
+from geovisio.web.utils import accountIdOrDefault
+from psycopg.types.json import Jsonb
+from geovisio.utils import db
+from geovisio.utils.params import validation_error
+from geovisio import errors
+from pydantic import BaseModel, ConfigDict, ValidationError
+from uuid import UUID
+from typing import List, Optional
+from flask import Blueprint, request, current_app
+from flask_babel import gettext as _
+
+
+bp = Blueprint("annotations", __name__, url_prefix="/api")

geovisio/web/auth.py

@@ -70,7 +70,7 @@ def auth():
     oauth_info = utils.auth.oauth_provider.get_user_oauth_info(tokenResponse)
     with db.cursor(current_app) as cursor:
         res = cursor.execute(
-            "INSERT INTO accounts (name, oauth_provider, oauth_id) VALUES (%(name)s, %(provider)s, %(id)s) ON CONFLICT (oauth_provider, oauth_id) DO UPDATE SET name = %(name)s RETURNING id, name",
+            "INSERT INTO accounts (name, oauth_provider, oauth_id) VALUES (%(name)s, %(provider)s, %(id)s) ON CONFLICT (oauth_provider, oauth_id) DO UPDATE SET name = %(name)s RETURNING id, name, tos_accepted",
             {
                 "provider": utils.auth.oauth_provider.name,
                 "id": oauth_info.id,
@@ -79,21 +79,25 @@ def auth():
         ).fetchone()
         if res is None:
             raise Exception("Impossible to insert user in database")
-        id, name = res
+        id, name, tos_accepted = res
         account = Account(
             id=str(id),  # convert uuid to string for serialization
             name=name,
             oauth_provider=utils.auth.oauth_provider.name,
             oauth_id=oauth_info.id,
+            tos_accepted=tos_accepted,
         )
         session[ACCOUNT_KEY] = account.model_dump(exclude_none=True)
         session.permanent = True
 
         next_url = session.pop(NEXT_URL_KEY, None)
-        if next_url:
-            response = flask.make_response(redirect(next_url))
+        if not tos_accepted and current_app.config["API_ENFORCE_TOS_ACCEPTANCE"]:
+            args = {"next_url": next_url} if next_url else None
+            next_url = current_app.config["API_WEBSITE_URL"].tos_validation_page(args)
         else:
-            response = flask.make_response(redirect("/"))
+            next_url = next_url or "/"
+
+        response = flask.make_response(redirect(next_url))
 
         # also store id/name in cookies for the front end to use those
         max_age = current_app.config["PERMANENT_SESSION_LIFETIME"]