geovisio 2.7.1__py3-none-any.whl → 2.8.1__py3-none-any.whl

geovisio/utils/upload_set.py

@@ -17,6 +17,8 @@ from flask import current_app
 from flask_babel import gettext as _
 from geopic_tag_reader import sequence as geopic_sequence, reader
 
+from geovisio.utils.loggers import getLoggerWithExtra
+
 
 class AggregatedStatus(BaseModel):
     """Aggregated status"""
@@ -147,6 +149,7 @@ class UploadSetFile(BaseModel):
     """File uploaded in an UploadSet"""
 
     picture_id: Optional[UUID] = None
+    """ID of the picture this file belongs to. Can only be seen by the owner of the File"""
     file_name: str
     content_md5: Optional[UUID] = None
     inserted_at: datetime
@@ -434,6 +437,7 @@ def dispatch(upload_set_id: UUID):
     if not db_upload_set:
         raise Exception(f"Upload set {upload_set_id} not found")
 
+    logger = getLoggerWithExtra("geovisio.upload_set", {"upload_set_id": str(upload_set_id)})
     with db.conn(current_app) as conn:
         with conn.transaction(), conn.cursor(row_factory=dict_row) as cursor:
 
@@ -448,16 +452,25 @@ def dispatch(upload_set_id: UUID):
     p.heading as heading,
     p.metadata->>'originalFileName' as file_name,
     p.metadata,
-    s.id as sequence_id
+    s.id as sequence_id,
+    f is null as has_no_file
 FROM pictures p
 LEFT JOIN sequences_pictures sp ON sp.pic_id = p.id
 LEFT JOIN sequences s ON s.id = sp.seq_id
+LEFT JOIN files f ON f.picture_id = p.id
 WHERE p.upload_set_id = %(upload_set_id)s"""
                 ),
                 {"upload_set_id": upload_set_id},
             ).fetchall()
 
+            # there is currently a bug where 2 pictures can be uploaded for the same file, so only 1 is associated to it.
+            # we want to delete one of them
+            # Those duplicates happen when a client send an upload that timeouts, but the client retries the upload and the server is not aware of this timeout (the connection is not closed).
+            # Note: later, if we are confident the bug has been removed, we might clean this code.
+            pics_to_delete_bug = [p["id"] for p in db_pics if p["has_no_file"]]
+            db_pics = [p for p in db_pics if p["has_no_file"] is False]  # pictures without files will be deleted, we don't need them
             pics_by_filename = {p["file_name"]: p for p in db_pics}
+
             pics = [
                 geopic_sequence.Picture(
                     p["file_name"],
@@ -483,16 +496,19 @@ WHERE p.upload_set_id = %(upload_set_id)s"""
                     maxDistance=db_upload_set.duplicate_distance, maxRotationAngle=db_upload_set.duplicate_rotation
                 ),
                 sortMethod=db_upload_set.sort_method,
-                splitParams=geopic_sequence.SplitParams(maxDistance=db_upload_set.split_distance, maxTime=db_upload_set.split_time.seconds),
+                splitParams=geopic_sequence.SplitParams(
+                    maxDistance=db_upload_set.split_distance, maxTime=db_upload_set.split_time.total_seconds()
+                ),
             )
             reused_sequence = set()
 
-            pics_to_delete = [pics_by_filename[p.filename]["id"] for p in report.duplicate_pictures or []]
+            pics_to_delete_duplicates = [pics_by_filename[p.filename]["id"] for p in report.duplicate_pictures or []]
+            pics_to_delete = pics_to_delete_duplicates + pics_to_delete_bug
             if pics_to_delete:
-                logging.debug(f"For uploadset '{upload_set_id}', nb duplicate pictures {len(pics_to_delete)}")
-                logging.debug(
-                    f"For uploadset '{upload_set_id}', duplicate pictures {[p.filename for p in report.duplicate_pictures or []]}"
+                logger.debug(
+                    f"nb duplicate pictures {len(pics_to_delete_duplicates)} {f' and {len(pics_to_delete_bug)} pictures without files' if pics_to_delete_bug else ''}"
                 )
+                logger.debug(f"duplicate pictures {[p.filename for p in report.duplicate_pictures or []]}")
 
                 cursor.execute(SQL("CREATE TEMPORARY TABLE tmp_duplicates(picture_id UUID) ON COMMIT DROP"))
                 with cursor.copy("COPY tmp_duplicates(picture_id) FROM stdin;") as copy:
@@ -507,16 +523,17 @@ WHERE p.upload_set_id = %(upload_set_id)s"""
                 # delete all pictures (the DB triggers will also add background jobs to delete the associated files)
                 cursor.execute(SQL("DELETE FROM pictures WHERE id IN (select picture_id FROM tmp_duplicates)"))
 
-            for s in report.sequences:
+            number_title = len(report.sequences) > 1
+            existing_sequences = set(p["sequence_id"] for p in db_pics if p["sequence_id"])
+            new_sequence_ids = set()
+            for i, s in enumerate(report.sequences, start=1):
                 existing_sequence = next(
                     (seq for p in s.pictures if (seq := pics_by_filename[p.filename]["sequence_id"]) not in reused_sequence),
                     None,
                 )
                 # if some of the pictures were already in a sequence, we should not create a new one
                 if existing_sequence:
-                    logging.info(
-                        f"For uploadset '{upload_set_id}', sequence {existing_sequence} already contains pictures, we will not create a new one"
-                    )
+                    logger.info(f"sequence {existing_sequence} already contains pictures, we will not create a new one")
                     # we should wipe the sequences_pictures though
                     seq_id = existing_sequence
                     cursor.execute(
@@ -525,6 +542,7 @@ WHERE p.upload_set_id = %(upload_set_id)s"""
                     )
                     reused_sequence.add(seq_id)
                 else:
+                    new_title = f"{db_upload_set.title}{f'-{i}' if number_title else ''}"
                     seq_id = cursor.execute(
                         SQL(
                             """INSERT INTO sequences(account_id, metadata, user_agent)
@@ -533,12 +551,14 @@ RETURNING id"""
                         ),
                         {
                             "account_id": db_upload_set.account_id,
-                            "metadata": Jsonb({"title": db_upload_set.title}),
+                            "metadata": Jsonb({"title": new_title}),
                             "user_agent": db_upload_set.user_agent,
                         },
                     ).fetchone()
                     seq_id = seq_id["id"]
 
+                new_sequence_ids.add(seq_id)
+
                 with cursor.copy("COPY sequences_pictures(seq_id, pic_id, rank) FROM stdin;") as copy:
                     for i, p in enumerate(s.pictures, 1):
                         copy.write_row(
@@ -547,8 +567,17 @@ RETURNING id"""
 
                 sequences.add_finalization_job(cursor=cursor, seqId=seq_id)
 
+            # we can delete all the old sequences
+            sequences_to_delete = existing_sequences - new_sequence_ids
+            if sequences_to_delete:
+                logger.debug(f"sequences to delete = {sequences_to_delete} (existing = {existing_sequences}, new = {new_sequence_ids})")
+                conn.execute(SQL("DELETE FROM sequences_pictures WHERE seq_id = ANY(%(seq_ids)s)"), {"seq_ids": list(sequences_to_delete)})
+                conn.execute(
+                    SQL("UPDATE sequences SET status = 'deleted' WHERE id = ANY(%(seq_ids)s)"), {"seq_ids": list(sequences_to_delete)}
+                )
+
             for s in report.sequences_splits or []:
-                logging.debug(f"For uploadset '{upload_set_id}', split = {s.prevPic.filename} -> {s.nextPic.filename} : {s.reason}")
+                logger.debug(f"split = {s.prevPic.filename} -> {s.nextPic.filename} : {s.reason}")
             conn.execute(SQL("UPDATE upload_sets SET dispatched = true WHERE id = %(upload_set_id)s"), {"upload_set_id": db_upload_set.id})
 
 
@@ -567,19 +596,39 @@ def insertFileInDatabase(
 ) -> UploadSetFile:
     """Insert a file linked to an UploadSet into the database"""
 
+    # we check if there is already a file with this name in the upload set with an associated picture.
+    # If there is no picture (because the picture has been rejected), we accept that the file is overridden
+    existing_file = cursor.execute(
+        SQL(
+            """SELECT picture_id, rejection_status
+            FROM files
+            WHERE upload_set_id = %(upload_set_id)s AND file_name = %(file_name)s AND picture_id IS NOT NULL"""
+        ),
+        params={
+            "upload_set_id": upload_set_id,
+            "file_name": file_name,
+        },
+    ).fetchone()
+    if existing_file:
+        raise errors.InvalidAPIUsage(
+            _("A different picture with the same name has already been added to this uploadset"),
+            status_code=409,
+            payload={"existing_item": {"id": existing_file["picture_id"]}},
+        )
+
     f = cursor.execute(
         SQL(
             """INSERT INTO files(
-        upload_set_id, picture_id, file_type, file_name,
-        size, content_md5, rejection_status, rejection_message, rejection_details)
-    VALUES (
-        %(upload_set_id)s, %(picture_id)s, %(type)s, %(file_name)s,
-        %(size)s, %(content_md5)s, %(rejection_status)s, %(rejection_message)s, %(rejection_details)s)
-    ON CONFLICT (upload_set_id, file_name)
-    DO UPDATE SET picture_id = %(picture_id)s, size = %(size)s, content_md5 = %(content_md5)s,
-        rejection_status = %(rejection_status)s, rejection_message = %(rejection_message)s, rejection_details = %(rejection_details)s
-    RETURNING *
-    """
+    upload_set_id, picture_id, file_type, file_name,
+    size, content_md5, rejection_status, rejection_message, rejection_details)
+VALUES (
+    %(upload_set_id)s, %(picture_id)s, %(type)s, %(file_name)s,
+    %(size)s, %(content_md5)s, %(rejection_status)s, %(rejection_message)s, %(rejection_details)s)
+ON CONFLICT (upload_set_id, file_name)
+DO UPDATE SET picture_id = %(picture_id)s, size = %(size)s, content_md5 = %(content_md5)s,
+    rejection_status = %(rejection_status)s, rejection_message = %(rejection_message)s, rejection_details = %(rejection_details)s
+WHERE files.picture_id IS NULL -- check again that we do not override an existing picture
+RETURNING *"""
         ),
         params={
             "upload_set_id": upload_set_id,

geovisio/utils/website.py

@@ -0,0 +1,53 @@
+from typing import Optional, Dict
+
+from flask import url_for
+
+from geovisio import web
+
+WEBSITE_UNDER_SAME_HOST = "same-host"
+
+TOKEN_ACCEPTED_PAGE = "token-accepted"
+TOS_VALIDATION_PAGE = "tos-validation"
+
+
+class Website:
+    """Website associated to the API.
+    This wrapper will define the routes we expect from the website.
+
+    We should limit the interraction from the api to the website, but for some flow (especially auth flows), it's can be useful to redirect to website's page
+
+    If the url is:
+    * set to `false`, there is no associated website
+    * set to `same-host`, the website is assumed to be on the same host as the API (and will respect the host of the current request)
+    * else it should be a valid url
+    """
+
+    def __init__(self, website_url: str):
+        if website_url == WEBSITE_UNDER_SAME_HOST:
+            self.url = WEBSITE_UNDER_SAME_HOST
+        elif website_url == "false":
+            self.url = None
+        elif website_url.startswith("http"):
+            self.url = website_url
+            if not self.url.endswith("/"):
+                self.url += "/"
+        else:
+            raise Exception(
+                "API_WEBSITE_URL should either be `same-host` (and the website will be assumed to be on the same host), set to `false` if there is no website, or a valid URL"
+            )
+
+    def _to_url(self, route: str, params: Optional[Dict[str, str]] = None):
+        if not self.url:
+            return None
+
+        base_url = self.url if self.url != WEBSITE_UNDER_SAME_HOST else url_for("index", _external=True)
+
+        from urllib.parse import urlencode
+
+        return f"{base_url}{route}{f'?{urlencode(params)}' if params else ''}"
+
+    def tos_validation_page(self, params: Optional[Dict[str, str]] = None):
+        return self._to_url(TOS_VALIDATION_PAGE, params)
+
+    def cli_token_accepted_page(self, params: Optional[Dict[str, str]] = None):
+        return self._to_url(TOKEN_ACCEPTED_PAGE, params)

geovisio/web/annotations.py

@@ -0,0 +1,17 @@
+from geovisio.utils import auth
+from psycopg.rows import dict_row, class_row
+from psycopg.sql import SQL
+from geovisio.utils.semantics import Entity, EntityType, SemanticTagUpdate, update_tags
+from geovisio.web.utils import accountIdOrDefault
+from psycopg.types.json import Jsonb
+from geovisio.utils import db
+from geovisio.utils.params import validation_error
+from geovisio import errors
+from pydantic import BaseModel, ConfigDict, ValidationError
+from uuid import UUID
+from typing import List, Optional
+from flask import Blueprint, request, current_app
+from flask_babel import gettext as _
+
+
+bp = Blueprint("annotations", __name__, url_prefix="/api")

geovisio/web/auth.py

@@ -70,7 +70,7 @@ def auth():
     oauth_info = utils.auth.oauth_provider.get_user_oauth_info(tokenResponse)
     with db.cursor(current_app) as cursor:
         res = cursor.execute(
-            "INSERT INTO accounts (name, oauth_provider, oauth_id) VALUES (%(name)s, %(provider)s, %(id)s) ON CONFLICT (oauth_provider, oauth_id) DO UPDATE SET name = %(name)s RETURNING id, name",
+            "INSERT INTO accounts (name, oauth_provider, oauth_id) VALUES (%(name)s, %(provider)s, %(id)s) ON CONFLICT (oauth_provider, oauth_id) DO UPDATE SET name = %(name)s RETURNING id, name, tos_accepted",
             {
                 "provider": utils.auth.oauth_provider.name,
                 "id": oauth_info.id,
@@ -79,21 +79,26 @@ def auth():
         ).fetchone()
         if res is None:
             raise Exception("Impossible to insert user in database")
-        id, name = res
+        id, name, tos_accepted = res
         account = Account(
             id=str(id),  # convert uuid to string for serialization
             name=name,
             oauth_provider=utils.auth.oauth_provider.name,
             oauth_id=oauth_info.id,
+            tos_accepted=tos_accepted,
         )
         session[ACCOUNT_KEY] = account.model_dump(exclude_none=True)
         session.permanent = True
 
         next_url = session.pop(NEXT_URL_KEY, None)
-        if next_url:
-            response = flask.make_response(redirect(next_url))
-        else:
-            response = flask.make_response(redirect("/"))
+        if not tos_accepted and current_app.config["API_ENFORCE_TOS_ACCEPTANCE"]:
+            args = {"next_url": next_url} if next_url else None
+            next_url = current_app.config["API_WEBSITE_URL"].tos_validation_page(args)
+
+        if next_url is None:
+            next_url = "/"
+
+        response = flask.make_response(redirect(next_url))
 
         # also store id/name in cookies for the front end to use those
         max_age = current_app.config["PERMANENT_SESSION_LIFETIME"]