geovisio 2.9.0__py3-none-any.whl → 2.11.0__py3-none-any.whl

geovisio/utils/upload_set.py

@@ -6,6 +6,7 @@ from geovisio.utils.extent import TemporalExtent
 from uuid import UUID
 from typing import Optional, List, Dict, Any
 from datetime import datetime, timedelta
+from dataclasses import dataclass
 from geovisio.utils import cql2, db, sequences
 from geovisio import errors
 from geovisio.utils.link import make_link, Link
@@ -16,6 +17,8 @@ from psycopg.rows import class_row, dict_row
 from flask import current_app
 from flask_babel import gettext as _
 from geopic_tag_reader import sequence as geopic_sequence, reader
+from geovisio.utils.tags import SemanticTag
+from geovisio.web.params import Visibility
 
 from geovisio.utils.loggers import getLoggerWithExtra
 
@@ -73,15 +76,27 @@ class UploadSet(BaseModel):
     title: str
     estimated_nb_files: Optional[int] = None
     sort_method: geopic_sequence.SortMethod
-    split_distance: int
-    split_time: timedelta
-    duplicate_distance: float
-    duplicate_rotation: int
+    no_split: Optional[bool] = None
+    split_distance: Optional[int] = None
+    split_time: Optional[timedelta] = None
+    no_deduplication: Optional[bool] = None
+    duplicate_distance: Optional[float] = None
+    duplicate_rotation: Optional[int] = None
     metadata: Optional[Dict[str, Any]]
     user_agent: Optional[str] = Field(exclude=True)
     associated_collections: List[AssociatedCollection] = []
     nb_items: int = 0
     items_status: Optional[AggregatedStatus] = None
+    semantics: List[SemanticTag] = Field(default_factory=list)
+    """Semantic tags associated to the upload_set"""
+    relative_heading: Optional[int] = None
+    """The relative heading (in degrees), offset based on movement path (0° = looking forward, -90° = looking left, 90° = looking right). For single picture upload_sets, 0° is heading north). Is applied to all associated collections if set."""
+    visibility: Optional[Visibility] = None
+    """Visibility of the upload set. Can be set to:
+    * `anyone`: the upload is visible to anyone
+    * `owner-only`: the upload is visible to the owner and administrator only
+    * `logged-only`: the upload is visible to logged users only   
+    """
 
     @computed_field
     @property
@@ -128,19 +143,13 @@ class FileRejectionStatus(Enum):
     """other_error means there was an error that is not related to the picture itself"""
 
 
-class FileRejectionDetails(BaseModel):
-
-    missing_fields: List[str]
-    """Mandatory metadata missing from the file. Metadata can be `datetime` or `location`."""
-
-
 class FileRejection(BaseModel):
     """Details about a file rejection"""
 
     reason: str
     severity: FileRejectionStatusSeverity
     message: Optional[str]
-    details: Optional[FileRejectionDetails]
+    details: Optional[Dict[str, Any]]
 
     model_config = ConfigDict(use_enum_values=True, use_attribute_docstrings=True)
 
@@ -224,32 +233,32 @@ def get_simple_upload_set(id: UUID) -> Optional[UploadSet]:
     return u
 
 
-def get_upload_set(id: UUID) -> Optional[UploadSet]:
+def get_upload_set(id: UUID, account_to_query: Optional[UUID] = None) -> Optional[UploadSet]:
     """Get the UploadSet corresponding to the ID"""
     db_upload_set = db.fetchone(
         current_app,
         SQL(
             """WITH picture_last_job AS (
     SELECT p.id as picture_id,
-        -- Note: to know if a picture is beeing processed, check the latest job_history entry for this picture
-        -- If there is no finished_at, the picture is still beeing processed
+        -- Note: to know if a picture is being processed, check the latest job_history entry for this picture
+        -- If there is no finished_at, the picture is still being processed
         (MAX(ARRAY [started_at, finished_at])) AS last_job,
         p.preparing_status,
         p.status,
         p.upload_set_id
     FROM pictures p
         LEFT JOIN job_history ON p.id = job_history.picture_id
-    WHERE p.upload_set_id = %(id)s
+    WHERE p.upload_set_id = %(id)s AND is_picture_visible_by_user(p, %(account_to_query)s)
     GROUP BY p.id
 ),
 picture_statuses AS (
-    SELECT 
+    SELECT
         *,
         (last_job[1] IS NOT NULL AND last_job[2] IS NULL) AS is_job_running
         FROM picture_last_job psj
 ),
 associated_collections AS (
-    SELECT 
+    SELECT
         ps.upload_set_id,
         COUNT(ps.picture_id) FILTER (WHERE ps.preparing_status = 'broken') AS nb_broken,
         COUNT(ps.picture_id) FILTER (WHERE ps.preparing_status = 'prepared') AS nb_prepared,
@@ -264,10 +273,19 @@ associated_collections AS (
     FROM picture_statuses ps
         JOIN sequences_pictures sp ON sp.pic_id = ps.picture_id
         JOIN sequences s ON s.id = sp.seq_id
-    WHERE ps.upload_set_id = %(id)s AND s.status != 'deleted'
+    WHERE ps.upload_set_id = %(id)s AND s.status != 'deleted' AND is_sequence_visible_by_user(s, %(account_to_query)s)
     GROUP BY ps.upload_set_id,
         s.id
 ),
+semantics AS (
+    SELECT upload_set_id, json_agg(json_strip_nulls(json_build_object(
+        'key', key,
+        'value', value
+    )) ORDER BY key, value) AS semantics
+    FROM upload_sets_semantics
+    WHERE upload_set_id = %(id)s
+    GROUP BY upload_set_id
+),
 upload_set_statuses AS (
     SELECT ps.upload_set_id,
         COUNT(ps.picture_id) AS nb_items,
@@ -280,13 +298,14 @@ upload_set_statuses AS (
 )
 SELECT u.*,
     COALESCE(us.nb_items, 0) AS nb_items,
+    COALESCE(s.semantics, '[]'::json) AS semantics,
     json_build_object(
         'broken', COALESCE(us.nb_broken, 0),
         'prepared', COALESCE(us.nb_prepared, 0),
         'not_processed', COALESCE(us.nb_not_processed, 0),
         'preparing', COALESCE(us.nb_preparing, 0),
         'rejected', (
-            SELECT count(*) FROM files 
+            SELECT count(*) FROM files
             WHERE upload_set_id = %(id)s AND rejection_status IS NOT NULL
         )
     ) AS items_status,
@@ -327,9 +346,10 @@ SELECT u.*,
     ) AS associated_collections
 FROM upload_sets u
 LEFT JOIN upload_set_statuses us on us.upload_set_id = u.id
-WHERE u.id = %(id)s"""
+LEFT JOIN semantics s on s.upload_set_id = u.id
+WHERE u.id = %(id)s AND is_upload_set_visible_by_user(u, %(account_to_query)s)"""
         ),
-        {"id": id},
+        {"id": id, "account_to_query": account_to_query},
         row_factory=class_row(UploadSet),
     )
 
@@ -360,44 +380,39 @@ def _parse_filter(filter: Optional[str]) -> SQL:
     return cql2.parse_cql2_filter(filter, FIELD_TO_SQL_FILTER)
 
 
-def list_upload_sets(account_id: UUID, limit: int = 100, filter: Optional[str] = None) -> UploadSets:
+def list_upload_sets(
+    account_id: UUID, limit: int = 100, filter: Optional[str] = None, account_to_query: Optional[UUID] = None
+) -> UploadSets:
     filter_sql = _parse_filter(filter)
     l = db.fetchall(
         current_app,
         SQL(
-            """SELECT 
+            """SELECT
             u.*,
             COALESCE(
                 (
-                    SELECT 
+                    SELECT
                         json_agg(json_build_object(
-                            'id', ac.collection_id,
-                            'nb_items', ac.nb_items
+                            'id', s.id,
+                            'nb_items', s.nb_pictures
                         ))
-                    FROM (
-                        SELECT 
-                            sp.seq_id as collection_id,
-                            count(sp.pic_id) AS nb_items
-                        FROM pictures p 
-                        JOIN sequences_pictures sp ON sp.pic_id = p.id
-                        WHERE p.upload_set_id = u.id
-                        GROUP BY sp.seq_id
-                    ) ac
+                    FROM sequences s
+                    WHERE s.upload_set_id = u.id
                 ),
                 '[]'::json
             ) AS associated_collections,
             (
                 SELECT count(*) AS nb
-                FROM pictures p 
+                FROM pictures p
                 WHERE p.upload_set_id = u.id
             ) AS nb_items
         FROM upload_sets u
-        WHERE account_id = %(account_id)s AND {filter}
+        WHERE account_id = %(account_id)s AND is_upload_set_visible_by_user(u, %(account_to_query)s) AND {filter}
         ORDER BY created_at ASC
         LIMIT %(limit)s
         """
         ).format(filter=filter_sql),
-        {"account_id": account_id, "limit": limit},
+        {"account_id": account_id, "limit": limit, "account_to_query": account_to_query},
         row_factory=class_row(UploadSet),
     )
 
@@ -408,7 +423,7 @@ def ask_for_dispatch(upload_set_id: UUID):
     """Add a dispatch task to the job queue for the upload set. If there is already a task, postpone it."""
     with db.conn(current_app) as conn:
         conn.execute(
-            """INSERT INTO 
+            """INSERT INTO
             job_queue(sequence_id, task)
             VALUES (%(upload_set_id)s, 'dispatch')
             ON CONFLICT (upload_set_id) DO UPDATE SET ts = CURRENT_TIMESTAMP""",
@@ -416,7 +431,13 @@ def ask_for_dispatch(upload_set_id: UUID):
         )
 
 
-def dispatch(upload_set_id: UUID):
+@dataclass
+class PicToDelete:
+    picture_id: UUID
+    detail: Optional[Dict] = None
+
+
+def dispatch(conn: psycopg.Connection, upload_set_id: UUID):
     """Finalize an upload set.
 
     For the moment we only create a collection around all the items of the upload set, but later we'll split the items into several collections
@@ -429,13 +450,15 @@ def dispatch(upload_set_id: UUID):
         raise Exception(f"Upload set {upload_set_id} not found")
 
     logger = getLoggerWithExtra("geovisio.upload_set", {"upload_set_id": str(upload_set_id)})
-    with db.conn(current_app) as conn:
-        with conn.transaction(), conn.cursor(row_factory=dict_row) as cursor:
-
-            # get all the pictures of the upload set
-            db_pics = cursor.execute(
-                SQL(
-                    """SELECT 
+    with conn.transaction(), conn.cursor(row_factory=dict_row) as cursor:
+        # we put a lock on the upload set, to avoid new semantics being added while dispatching it
+        # Note: I did not find a way to only put a lock on the upload_sets_semantics table, so we lock the whole upload_set row (and any child rows)
+        _us_lock = cursor.execute(SQL("SELECT id FROM upload_sets WHERE id = %s FOR UPDATE"), [upload_set_id])
+
+        # get all the pictures of the upload set
+        db_pics = cursor.execute(
+            SQL(
+                """SELECT
     p.id,
     p.ts,
     ST_X(p.geom) as lon,
@@ -444,132 +467,198 @@ def dispatch(upload_set_id: UUID):
     p.metadata->>'originalFileName' as file_name,
     p.metadata,
     s.id as sequence_id,
-    f is null as has_no_file
+    f is null as has_no_file,
+    p.heading_computed
 FROM pictures p
 LEFT JOIN sequences_pictures sp ON sp.pic_id = p.id
 LEFT JOIN sequences s ON s.id = sp.seq_id
 LEFT JOIN files f ON f.picture_id = p.id
 WHERE p.upload_set_id = %(upload_set_id)s"""
-                ),
-                {"upload_set_id": upload_set_id},
-            ).fetchall()
-
-            # there is currently a bug where 2 pictures can be uploaded for the same file, so only 1 is associated to it.
-            # we want to delete one of them
-            # Those duplicates happen when a client send an upload that timeouts, but the client retries the upload and the server is not aware of this timeout (the connection is not closed).
-            # Note: later, if we are confident the bug has been removed, we might clean this code.
-            pics_to_delete_bug = [p["id"] for p in db_pics if p["has_no_file"]]
-            db_pics = [p for p in db_pics if p["has_no_file"] is False]  # pictures without files will be deleted, we don't need them
-            pics_by_filename = {p["file_name"]: p for p in db_pics}
-
-            pics = [
-                geopic_sequence.Picture(
-                    p["file_name"],
-                    reader.GeoPicTags(
-                        lon=p["lon"],
-                        lat=p["lat"],
-                        ts=p["ts"],
-                        type=p["metadata"]["type"],
-                        heading=p["heading"],
-                        make=p["metadata"]["make"],
-                        model=p["metadata"]["model"],
-                        focal_length=p["metadata"]["focal_length"],
-                        crop=p["metadata"]["crop"],
-                        exif={},
-                    ),
-                )
-                for p in db_pics
-            ]
+            ),
+            {"upload_set_id": upload_set_id},
+        ).fetchall()
 
-            report = geopic_sequence.dispatch_pictures(
-                pics,
-                mergeParams=geopic_sequence.MergeParams(
-                    maxDistance=db_upload_set.duplicate_distance, maxRotationAngle=db_upload_set.duplicate_rotation
-                ),
-                sortMethod=db_upload_set.sort_method,
-                splitParams=geopic_sequence.SplitParams(
-                    maxDistance=db_upload_set.split_distance, maxTime=db_upload_set.split_time.total_seconds()
+        config = cursor.execute(
+            SQL(
+                "SELECT default_split_distance, default_split_time, default_duplicate_distance, default_duplicate_rotation FROM configurations"
+            )
+        ).fetchone()
+
+        # there is currently a bug where 2 pictures can be uploaded for the same file, so only 1 is associated to it.
+        # we want to delete one of them
+        # Those duplicates happen when a client send an upload that timeouts, but the client retries the upload and the server is not aware of this timeout (the connection is not closed).
+        # Note: later, if we are confident the bug has been removed, we might clean this code.
+        pics_to_delete_bug = [PicToDelete(picture_id=p["id"]) for p in db_pics if p["has_no_file"]]
+        db_pics = [p for p in db_pics if p["has_no_file"] is False]  # pictures without files will be deleted, we don't need them
+        pics_by_filename = {p["file_name"]: p for p in db_pics}
+
+        pics = [
+            geopic_sequence.Picture(
+                p["file_name"],
+                reader.GeoPicTags(
+                    lon=p["lon"],
+                    lat=p["lat"],
+                    ts=p["ts"],
+                    type=p["metadata"]["type"],
+                    heading=p["heading"],
+                    make=p["metadata"]["make"],
+                    model=p["metadata"]["model"],
+                    focal_length=p["metadata"]["focal_length"],
+                    crop=p["metadata"]["crop"],
+                    exif={},
                 ),
+                heading_computed=p["heading_computed"],
             )
-            reused_sequence = set()
+            for p in db_pics
+        ]
 
-            pics_to_delete_duplicates = [pics_by_filename[p.filename]["id"] for p in report.duplicate_pictures or []]
-            pics_to_delete = pics_to_delete_duplicates + pics_to_delete_bug
-            if pics_to_delete:
-                logger.debug(
-                    f"nb duplicate pictures {len(pics_to_delete_duplicates)} {f' and {len(pics_to_delete_bug)} pictures without files' if pics_to_delete_bug else ''}"
-                )
-                logger.debug(f"duplicate pictures {[p.filename for p in report.duplicate_pictures or []]}")
+        split_params = None
+        if not db_upload_set.no_split:
+            distance = db_upload_set.split_distance if db_upload_set.split_distance is not None else config["default_split_distance"]
+            t = db_upload_set.split_time if db_upload_set.split_time is not None else config["default_split_time"]
+            if t is not None and distance is not None:
+                split_params = geopic_sequence.SplitParams(maxDistance=distance, maxTime=t.total_seconds())
+        merge_params = None
+        if not db_upload_set.no_deduplication:
+            distance = (
+                db_upload_set.duplicate_distance if db_upload_set.duplicate_distance is not None else config["default_duplicate_distance"]
+            )
+            rotation = (
+                db_upload_set.duplicate_rotation if db_upload_set.duplicate_rotation is not None else config["default_duplicate_rotation"]
+            )
+            if distance is not None and rotation is not None:
+                merge_params = geopic_sequence.MergeParams(maxDistance=distance, maxRotationAngle=rotation)
 
-                cursor.execute(SQL("CREATE TEMPORARY TABLE tmp_duplicates(picture_id UUID) ON COMMIT DROP"))
-                with cursor.copy("COPY tmp_duplicates(picture_id) FROM stdin;") as copy:
-                    for p in pics_to_delete:
-                        copy.write_row((p,))
+        report = geopic_sequence.dispatch_pictures(
+            pics, mergeParams=merge_params, sortMethod=db_upload_set.sort_method, splitParams=split_params
+        )
+        reused_sequence = set()
+
+        pics_to_delete_duplicates = [
+            PicToDelete(
+                picture_id=pics_by_filename[d.picture.filename]["id"],
+                detail={
+                    "duplicate_of": str(pics_by_filename[d.duplicate_of.filename]["id"]),
+                    "distance": d.distance,
+                    "angle": d.angle,
+                },
+            )
+            for d in report.duplicate_pictures
+        ]
+        pics_to_delete = pics_to_delete_duplicates + pics_to_delete_bug
+        if pics_to_delete:
+            logger.debug(
+                f"nb duplicate pictures {len(pics_to_delete_duplicates)} {f' and {len(pics_to_delete_bug)} pictures without files' if pics_to_delete_bug else ''}"
+            )
+            logger.debug(f"duplicate pictures {[p.picture.filename for p in report.duplicate_pictures]}")
 
-                cursor.execute(
-                    SQL(
-                        "UPDATE files SET rejection_status = 'capture_duplicate' WHERE picture_id IN (select picture_id from tmp_duplicates)"
-                    )
+            cursor.execute(SQL("CREATE TEMPORARY TABLE tmp_duplicates(picture_id UUID, details JSONB) ON COMMIT DROP"))
+            with cursor.copy("COPY tmp_duplicates(picture_id, details) FROM stdin;") as copy:
+                for p in pics_to_delete:
+                    copy.write_row((p.picture_id, Jsonb(p.detail) if p.detail else None))
+
+            cursor.execute(
+                SQL(
+                    """UPDATE files SET 
+    rejection_status = 'capture_duplicate', rejection_details = d.details 
+FROM tmp_duplicates d 
+WHERE d.picture_id = files.picture_id"""
                 )
-                # delete all pictures (the DB triggers will also add background jobs to delete the associated files)
-                cursor.execute(SQL("DELETE FROM pictures WHERE id IN (select picture_id FROM tmp_duplicates)"))
-
-            number_title = len(report.sequences) > 1
-            existing_sequences = set(p["sequence_id"] for p in db_pics if p["sequence_id"])
-            new_sequence_ids = set()
-            for i, s in enumerate(report.sequences, start=1):
-                existing_sequence = next(
-                    (seq for p in s.pictures if (seq := pics_by_filename[p.filename]["sequence_id"]) not in reused_sequence),
-                    None,
+            )
+            # set all the pictures as waiting for deletion and add background jobs to delete them
+            # Note: we do not delte the picture's row because it can cause some deadlocks if some workers are preparing thoses pictures.
+            cursor.execute(SQL("UPDATE pictures SET status = 'waiting-for-delete' WHERE id IN (select picture_id FROM tmp_duplicates)"))
+            cursor.execute(
+                SQL(
+                    """INSERT INTO job_queue(picture_to_delete_id, task)
+    SELECT picture_id, 'delete' FROM tmp_duplicates"""
+                )
+            )
+
+        number_title = len(report.sequences) > 1
+        existing_sequences = set(p["sequence_id"] for p in db_pics if p["sequence_id"])
+        new_sequence_ids = set()
+        for i, s in enumerate(report.sequences, start=1):
+            existing_sequence = next(
+                (seq for p in s.pictures if (seq := pics_by_filename[p.filename]["sequence_id"]) not in reused_sequence),
+                None,
+            )
+            # if some of the pictures were already in a sequence, we should not create a new one
+            if existing_sequence:
+                logger.info(f"sequence {existing_sequence} already contains pictures, we will not create a new one")
+                # we should wipe the sequences_pictures though
+                seq_id = existing_sequence
+                cursor.execute(
+                    SQL("DELETE FROM sequences_pictures WHERE seq_id = %(seq_id)s"),
+                    {"seq_id": seq_id},
                 )
-                # if some of the pictures were already in a sequence, we should not create a new one
-                if existing_sequence:
-                    logger.info(f"sequence {existing_sequence} already contains pictures, we will not create a new one")
-                    # we should wipe the sequences_pictures though
-                    seq_id = existing_sequence
-                    cursor.execute(
-                        SQL("DELETE FROM sequences_pictures WHERE seq_id = %(seq_id)s"),
-                        {"seq_id": seq_id},
+                reused_sequence.add(seq_id)
+                # Note: we do not update the sequences_semantics if reusing a sequence, because the sequence semantics's updates are reported to the existing sequences if there are some
+            else:
+                new_title = f"{db_upload_set.title}{f'-{i}' if number_title else ''}"
+                seq_id = cursor.execute(
+                    SQL(
+                        """INSERT INTO sequences(account_id, metadata, user_agent, upload_set_id, visibility)
+                VALUES (%(account_id)s, %(metadata)s, %(user_agent)s, %(upload_set_id)s, %(visibility)s)
+                RETURNING id"""
+                    ),
+                    {
+                        "account_id": db_upload_set.account_id,
+                        "metadata": Jsonb({"title": new_title}),
+                        "user_agent": db_upload_set.user_agent,
+                        "upload_set_id": db_upload_set.id,
+                        "visibility": db_upload_set.visibility,
+                    },
+                ).fetchone()
+                seq_id = seq_id["id"]
+
+                # Pass all semantics to the new sequence
+                copy_upload_set_semantics_to_sequence(cursor, db_upload_set.id, seq_id)
+            new_sequence_ids.add(seq_id)
+
+            with cursor.copy("COPY sequences_pictures(seq_id, pic_id, rank) FROM stdin;") as copy:
+                for i, p in enumerate(s.pictures, 1):
+                    copy.write_row(
+                        (seq_id, pics_by_filename[p.filename]["id"], i),
                     )
-                    reused_sequence.add(seq_id)
-                else:
-                    new_title = f"{db_upload_set.title}{f'-{i}' if number_title else ''}"
-                    seq_id = cursor.execute(
-                        SQL(
-                            """INSERT INTO sequences(account_id, metadata, user_agent)
-VALUES (%(account_id)s, %(metadata)s, %(user_agent)s)
-RETURNING id"""
-                        ),
-                        {
-                            "account_id": db_upload_set.account_id,
-                            "metadata": Jsonb({"title": new_title}),
-                            "user_agent": db_upload_set.user_agent,
-                        },
-                    ).fetchone()
-                    seq_id = seq_id["id"]
-
-                new_sequence_ids.add(seq_id)
-
-                with cursor.copy("COPY sequences_pictures(seq_id, pic_id, rank) FROM stdin;") as copy:
-                    for i, p in enumerate(s.pictures, 1):
-                        copy.write_row(
-                            (seq_id, pics_by_filename[p.filename]["id"], i),
-                        )
 
-                sequences.add_finalization_job(cursor=cursor, seqId=seq_id)
+            sequences.add_finalization_job(cursor=cursor, seqId=seq_id)
 
-            # we can delete all the old sequences
-            sequences_to_delete = existing_sequences - new_sequence_ids
-            if sequences_to_delete:
-                logger.debug(f"sequences to delete = {sequences_to_delete} (existing = {existing_sequences}, new = {new_sequence_ids})")
-                conn.execute(SQL("DELETE FROM sequences_pictures WHERE seq_id = ANY(%(seq_ids)s)"), {"seq_ids": list(sequences_to_delete)})
-                conn.execute(
-                    SQL("UPDATE sequences SET status = 'deleted' WHERE id = ANY(%(seq_ids)s)"), {"seq_ids": list(sequences_to_delete)}
-                )
+        # we can delete all the old sequences
+        sequences_to_delete = existing_sequences - new_sequence_ids
+        if sequences_to_delete:
+            logger.debug(f"sequences to delete = {sequences_to_delete} (existing = {existing_sequences}, new = {new_sequence_ids})")
+            conn.execute(SQL("DELETE FROM sequences_pictures WHERE seq_id = ANY(%(seq_ids)s)"), {"seq_ids": list(sequences_to_delete)})
+            conn.execute(SQL("UPDATE sequences SET status = 'deleted' WHERE id = ANY(%(seq_ids)s)"), {"seq_ids": list(sequences_to_delete)})
 
-            for s in report.sequences_splits or []:
-                logger.debug(f"split = {s.prevPic.filename} -> {s.nextPic.filename} : {s.reason}")
-            conn.execute(SQL("UPDATE upload_sets SET dispatched = true WHERE id = %(upload_set_id)s"), {"upload_set_id": db_upload_set.id})
+        for s in report.sequences_splits or []:
+            logger.debug(f"split = {s.prevPic.filename} -> {s.nextPic.filename} : {s.reason}")
+        conn.execute(SQL("UPDATE upload_sets SET dispatched = true WHERE id = %(upload_set_id)s"), {"upload_set_id": db_upload_set.id})
+
+
+def copy_upload_set_semantics_to_sequence(cursor, db_upload_id: UUID, seq_id: UUID):
+    cursor.execute(
+        SQL(
+            """WITH upload_set_semantics AS (
+                SELECT key, value, upload_set_id, account_id
+                FROM upload_sets_semantics
+                WHERE upload_set_id = %(upload_set_id)s
+            ),
+            seq_sem AS (
+                INSERT INTO sequences_semantics(sequence_id, key, value)
+                SELECT %(seq_id)s, key, value
+                FROM upload_set_semantics
+            )
+            INSERT INTO sequences_semantics_history(sequence_id, account_id, ts, updates)
+            SELECT %(seq_id)s, account_id, NOW(), jsonb_build_object('key', key, 'value', value, 'action', 'add')
+            FROM upload_set_semantics
+            """
+        ),
+        {
+            "upload_set_id": db_upload_id,
+            "seq_id": seq_id,
+        },
+    )
 
 
 def insertFileInDatabase(
@@ -589,51 +678,59 @@ def insertFileInDatabase(
 
     # we check if there is already a file with this name in the upload set with an associated picture.
     # If there is no picture (because the picture has been rejected), we accept that the file is overridden
-    existing_file = cursor.execute(
-        SQL(
-            """SELECT picture_id, rejection_status
-            FROM files
-            WHERE upload_set_id = %(upload_set_id)s AND file_name = %(file_name)s AND picture_id IS NOT NULL"""
-        ),
-        params={
-            "upload_set_id": upload_set_id,
-            "file_name": file_name,
-        },
-    ).fetchone()
-    if existing_file:
-        raise errors.InvalidAPIUsage(
-            _("A different picture with the same name has already been added to this uploadset"),
-            status_code=409,
-            payload={"existing_item": {"id": existing_file["picture_id"]}},
-        )
+    with cursor.connection.transaction():
+        existing_file = cursor.execute(
+            SQL(
+                """SELECT picture_id, rejection_status
+                FROM files
+                WHERE upload_set_id = %(upload_set_id)s AND file_name = %(file_name)s AND picture_id IS NOT NULL"""
+            ),
+            params={
+                "upload_set_id": upload_set_id,
+                "file_name": file_name,
+            },
+        ).fetchone()
+        if existing_file:
+            raise errors.InvalidAPIUsage(
+                _("A different picture with the same name has already been added to this uploadset"),
+                status_code=409,
+                payload={"existing_item": {"id": existing_file["picture_id"]}},
+            )
 
-    f = cursor.execute(
-        SQL(
-            """INSERT INTO files(
-    upload_set_id, picture_id, file_type, file_name,
-    size, content_md5, rejection_status, rejection_message, rejection_details)
-VALUES (
-    %(upload_set_id)s, %(picture_id)s, %(type)s, %(file_name)s,
-    %(size)s, %(content_md5)s, %(rejection_status)s, %(rejection_message)s, %(rejection_details)s)
-ON CONFLICT (upload_set_id, file_name)
-DO UPDATE SET picture_id = %(picture_id)s, size = %(size)s, content_md5 = %(content_md5)s,
-    rejection_status = %(rejection_status)s, rejection_message = %(rejection_message)s, rejection_details = %(rejection_details)s
-WHERE files.picture_id IS NULL -- check again that we do not override an existing picture
-RETURNING *"""
-        ),
-        params={
-            "upload_set_id": upload_set_id,
-            "type": file_type,
-            "picture_id": picture_id,
-            "file_name": file_name,
-            "size": size,
-            "content_md5": content_md5,
-            "rejection_status": rejection_status,
-            "rejection_message": rejection_message,
-            "rejection_details": Jsonb(rejection_details),
-        },
-    )
-    return UploadSetFile(**f.fetchone())
+        f = cursor.execute(
+            SQL(
+                """INSERT INTO files(
+        upload_set_id, picture_id, file_type, file_name,
+        size, content_md5, rejection_status, rejection_message, rejection_details)
+    VALUES (
+        %(upload_set_id)s, %(picture_id)s, %(type)s, %(file_name)s,
+        %(size)s, %(content_md5)s, %(rejection_status)s, %(rejection_message)s, %(rejection_details)s)
+    ON CONFLICT (upload_set_id, file_name)
+    DO UPDATE SET picture_id = %(picture_id)s, size = %(size)s, content_md5 = %(content_md5)s,
+        rejection_status = %(rejection_status)s, rejection_message = %(rejection_message)s, rejection_details = %(rejection_details)s
+    WHERE files.picture_id IS NULL -- check again that we do not override an existing picture
+    RETURNING *"""
+            ),
+            params={
+                "upload_set_id": upload_set_id,
+                "type": file_type,
+                "picture_id": picture_id,
+                "file_name": file_name,
+                "size": size,
+                "content_md5": content_md5,
+                "rejection_status": rejection_status,
+                "rejection_message": rejection_message,
+                "rejection_details": Jsonb(rejection_details),
+            },
+        )
+        u = f.fetchone()
+        if u is None:
+            logging.error(f"Impossible to add file {file_name} to uploadset {upload_set_id}")
+            raise errors.InvalidAPIUsage(
+                _("Impossible to add the picture to this uploadset"),
+                status_code=500,
+            )
+        return UploadSetFile(**u)
 
 
 def get_upload_set_files(upload_set_id: UUID) -> UploadSetFiles:
@@ -642,15 +739,15 @@ def get_upload_set_files(upload_set_id: UUID) -> UploadSetFiles:
         current_app,
         SQL(
             """SELECT
-    upload_set_id, 
-    file_type, 
-    file_name, 
-    size, 
-    content_md5, 
+    upload_set_id,
+    file_type,
+    file_name,
+    size,
+    content_md5,
     rejection_status,
     rejection_message,
     rejection_details,
-    picture_id, 
+    picture_id,
     inserted_at
 FROM files
 WHERE upload_set_id = %(upload_set_id)s