geovisio 2.6.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

geovisio/workers/runner_pictures.py

@@ -1,25 +1,23 @@
-from fs import open_fs
 from fs.path import dirname
 from PIL import Image, ImageOps
 from flask import current_app
+from geovisio import utils
+from geovisio.utils import db, sequences, upload_set
 import psycopg
+from psycopg.rows import dict_row
 from psycopg.sql import SQL
 import sentry_sdk
-from geovisio import utils
 from geovisio import errors
 from dataclasses import dataclass
 import logging
 from contextlib import contextmanager
 from enum import Enum
-from typing import Any
+from typing import Any, Optional
 import threading
 from uuid import UUID
 from croniter import croniter
-from typing import Optional
 from datetime import datetime, timezone
-
 import geovisio.utils.filesystems
-from geovisio.utils.sequences import update_headings
 
 log = logging.getLogger("geovisio.runner_pictures")
 
@@ -27,7 +25,7 @@ PICTURE_PROCESS_MAX_RETRY = 10  # Number of times a job will be retryed if there
 
 
 class PictureBackgroundProcessor(object):
-    def init_app(self, app):
+    def __init__(self, app):
         nb_threads = app.config["EXECUTOR_MAX_WORKERS"]
         self.enabled = nb_threads != 0
 
@@ -38,7 +36,7 @@ class PictureBackgroundProcessor(object):
         else:
             import sys
 
-            if "run" in sys.argv or "waitress" in sys.argv:  # hack not to display a frightening warning uselessly
+            if "run" in sys.argv or "waitress" in sys.argv or "gunicorn" in sys.argv:  # hack not to display a frightening warning uselessly
                 log.warning("No picture background processor run, no picture will be processed unless another separate worker is run")
                 log.warning("A separate process can be run with:")
                 log.warning("flask picture-worker")
@@ -48,34 +46,63 @@ class PictureBackgroundProcessor(object):
         Ask for a background picture process that will run until not pictures need to be processed
         """
         if self.enabled:
-            worker = PictureProcessor(config=current_app.config)
-            return self.executor.submit(worker.process_next_pictures)
+            worker = PictureProcessor(app=current_app)
+            return self.executor.submit(worker.process_jobs)
 
 
-background_processor = PictureBackgroundProcessor()
+# background_processor = PictureBackgroundProcessor()
 
 
 class ProcessTask(str, Enum):
     prepare = "prepare"
     delete = "delete"
+    dispatch = "dispatch"
+    finalize = "finalize"
 
 
 @dataclass
 class DbPicture:
-    id: str
+    id: UUID
     metadata: dict
 
     def blurred_by_author(self):
         return self.metadata.get("blurredByAuthor", False)
 
 
+@dataclass
+class DbSequence:
+    id: UUID
+
+
+@dataclass
+class DbUploadSet:
+    id: UUID
+
+
 @dataclass
 class DbJob:
     reporting_conn: psycopg.Connection
-    id: UUID
-    pic: DbPicture
+    job_history_id: UUID  # ID of the job in the job_history
+    job_queue_id: UUID  # ID in the job_queue
+    pic: Optional[DbPicture]
+    upload_set: Optional[DbUploadSet]
+    seq: Optional[DbSequence]
+
     task: ProcessTask
 
+    def label(self):
+        impacted_object = ""
+        if self.pic:
+            impacted_object = f"picture {self.pic.id}"
+        elif self.seq:
+            impacted_object = f"sequence {self.seq.id}"
+        elif self.upload_set:
+            impacted_object = f"upload set {self.upload_set.id}"
+        else:
+            impacted_object = "unknown object"
+
+        return f"{self.task} for {impacted_object}"
+
 
 def processPictureFiles(pic: DbPicture, config):
     """Generates the files associated with a sequence picture.
@@ -91,7 +118,7 @@ def processPictureFiles(pic: DbPicture, config):
     config : dict
             Flask app.config (passed as param to allow using ThreadPoolExecutor)
     """
-    skipBlur = pic.blurred_by_author() or config.get("API_BLUR_URL") == None
+    skipBlur = pic.blurred_by_author() or config.get("API_BLUR_URL") is None
     fses = config["FILESYSTEMS"]
     fs = fses.permanent if skipBlur else fses.tmp
     picHdPath = utils.pictures.getHDPicturePath(pic.id)
@@ -156,31 +183,48 @@ class RecoverableProcessException(Exception):
         super().__init__(msg)
 
 
+class RetryLaterProcessException(Exception):
+    """Exception raised when we want to retry later, even if it's not an error"""
+
+    def __init__(self, msg):
+        super().__init__(msg)
+
+
 class PictureProcessor:
     stop: bool
     config: dict[Any, Any]
+    waiting_time: float
 
-    def __init__(self, config, stop=True) -> None:
-        self.config = config
+    def __init__(self, app, stop=True) -> None:
+        self.app = app
         self.stop = stop
         if threading.current_thread() is threading.main_thread():
             # if worker is in daemon mode, register signals to gracefully stop it
             self._register_signals()
         self.next_periodic_task_dt = None
+        self.cron = croniter(self.app.config["PICTURE_PROCESS_REFRESH_CRON"])
 
-    def process_next_pictures(self):
+        # Note: in tests, we don't want to wait between each picture processing
+        waiting_time = 0 if app.config.get("TESTING") is True else 1
+        self.waiting_time = waiting_time
+
+    def process_jobs(self):
         try:
-            while True:
-                self.check_periodic_tasks()
-                r = process_next_picture(self.config)
-                if not r:
-                    if self.stop:
+            with self.app.app_context():
+                while True:
+                    if self.app.pool.closed and self.stop:
+                        # in some tests, the pool is closed before the worker is stopped, we check this here
                         return
-                    # no more picture to process
-                    # wait a bit until there are some
-                    import time
+                    self.check_periodic_tasks()
+                    r = process_next_job(self.app)
+                    if not r:
+                        if self.stop:
+                            return
+                        # no more picture to process
+                        # wait a bit until there are some
+                        import time
 
-                    time.sleep(1)
+                        time.sleep(self.waiting_time)
 
         except:
             log.exception("Exiting thread")
@@ -200,15 +244,24 @@ class PictureProcessor:
         Check if a periodic task needs to be done, and do it if necessary
         This method ensure only one picture worker will do the needed periodic task
         """
-        with psycopg.connect(self.config["DB_URL"], autocommit=True) as db:
-            if self.next_periodic_task_dt is None:
-                self.next_periodic_task_dt = self.get_next_periodic_task_dt(db)
-
-            if datetime.now(timezone.utc) >= self.next_periodic_task_dt:
-                self.refresh_database(db)
-
-    def get_next_periodic_task_dt(self, db) -> datetime:
-        r = db.execute("SELECT refreshed_at, NOW() FROM refresh_database").fetchone()
+        if self.next_periodic_task_dt is None:
+            with db.conn(self.app) as conn:
+                self.next_periodic_task_dt = self.get_next_periodic_task_dt(conn)
+
+        if datetime.now(timezone.utc) >= self.next_periodic_task_dt:
+            with db.conn(self.app) as conn:
+                # since the next_periodic_task_dt can have been changed by another process, we check again that the task needs to be done
+                self.next_periodic_task_dt = self.get_next_periodic_task_dt(conn)
+                if datetime.now(timezone.utc) >= self.next_periodic_task_dt:
+                    if not self.refresh_database():
+                        # another refresh is in progress, we'll check again later and ask for the next refresh date considering it's in progress
+                        self.next_periodic_task_dt = self.cron.get_next(datetime, datetime.now(timezone.utc))
+                        logging.getLogger("geovisio.periodic_task").info(
+                            f"Refresh in progress, checking after = {self.next_periodic_task_dt}"
+                        )
+
+    def get_next_periodic_task_dt(self, conn) -> datetime:
+        r = conn.execute("SELECT refreshed_at, NOW() FROM refresh_database").fetchone()
         assert r  # the table always has exactly one row
 
         refreshed_at, db_time = r
@@ -216,42 +269,47 @@ class PictureProcessor:
         if refreshed_at is None:
             # if the db has never been updated, we need to update it now
             return current_time
-
-        cron = croniter(self.config["PICTURE_PROCESS_REFRESH_CRON"])
-
-        next_schedule_date = cron.get_next(datetime, refreshed_at)
+        next_schedule_date = self.cron.get_next(datetime, refreshed_at)
 
         # if the db time and the app time is not the same, we need to apply an offset on the scheduled time
         next_schedule_date += db_time - current_time
         logging.getLogger("geovisio.periodic_task").info(f"Next database refresh = {next_schedule_date}")
         return next_schedule_date
 
-    def refresh_database(self, db):
+    def refresh_database(self):
         with sentry_sdk.start_transaction(op="task", name="refresh_database"):
             # Note: there is a mechanism in `sequences.update_pictures_grid` to ensure that only one refresh can be done at one time, and it will update the `refreshed_at` value
-            updated = utils.sequences.update_pictures_grid(db)
-            if updated:
-                self.next_periodic_task_dt = self.get_next_periodic_task_dt(db)
-            else:
-                # no update could be done because another process was doing it, check next time the scheduled time
-                self.next_periodic_task_dt = None
+            return utils.sequences.update_pictures_grid()
 
 
-def process_next_picture(config):
+def process_next_job(app):
     with sentry_sdk.start_transaction(op="task", name="process_next_picture"):
-        with _get_next_picture_to_process(config) as job:
+        with _get_next_job(app) as job:
             if job is None:
                 return False
-            if job.task == ProcessTask.prepare:
+            if job.task == ProcessTask.prepare and job.pic:
                 with sentry_sdk.start_span(description="Processing picture") as span:
                     span.set_data("pic_id", job.pic.id)
                     with utils.time.log_elapsed(f"Processing picture {job.pic.id}"):
-                        processPictureFiles(job.pic, config)
-            elif job.task == ProcessTask.delete:
+                        # open another connection for reporting and queries
+                        processPictureFiles(job.pic, app.config)
+            elif job.task == ProcessTask.delete and job.pic:
                 with sentry_sdk.start_span(description="Deleting picture") as span:
                     span.set_data("pic_id", job.pic.id)
                     with utils.time.log_elapsed(f"Deleting picture {job.pic.id}"):
-                        _delete_picture(job)
+                        _delete_picture(job.pic)
+            elif job.task == ProcessTask.delete and job.upload_set:
+                with sentry_sdk.start_span(description="Deleting upload set") as span:
+                    span.set_data("us_id", job.upload_set.id)
+                    with utils.time.log_elapsed(f"Deleting upload set {job.upload_set.id}"):
+                        _delete_upload_set(job.upload_set)
+            elif job.task == ProcessTask.dispatch and job.upload_set:
+                with utils.time.log_elapsed(f"Dispatching upload set {job.upload_set.id}"):
+                    upload_set.dispatch(job.upload_set.id)
+            elif job.task == ProcessTask.finalize and job.seq:
+                with utils.time.log_elapsed(f"Finalizing sequence {job.seq.id}"):
+                    with job.reporting_conn.cursor(row_factory=dict_row) as cursor:
+                        sequences.finalize(cursor, job.seq.id)
             else:
                 raise RecoverableProcessException(f"Unhandled process task: {job.task}")
 
@@ -259,58 +317,67 @@ def process_next_picture(config):
 
 
 @contextmanager
-def _get_next_picture_to_process(config):
+def _get_next_job(app):
     """
-    Open a new connection and return the next picture to process
-    Note: the picture should be used as a context manager to close the connection when we stop using the returned picture.
+    Open a new connection and return the next job to process
+    Note: the job should be used as a context manager to close the connection when we stop using the returned job.
 
-    The new connection is needed because we lock the `pictures_to_process` for the whole transaction for another worker not to process the same picture
+    The new connection is needed because we lock the `job_queue` for the whole transaction for another worker not to process the same job
     """
     error = None
-    with psycopg.connect(config["DB_URL"], autocommit=True) as locking_transaction:
-        with locking_transaction.transaction():
-            r = locking_transaction.execute(
-                """
-            SELECT p.id, pictures_to_process.task, p.metadata
-                FROM pictures_to_process
-                JOIN pictures p ON p.id = pictures_to_process.picture_id
+    with app.pool.connection() as locking_transaction:
+        with locking_transaction.transaction(), locking_transaction.cursor(row_factory=dict_row) as cursor:
+            r = cursor.execute(
+                """SELECT j.id, j.picture_id, j.upload_set_id, j.sequence_id, j.task, p.metadata
+                FROM job_queue j
+                LEFT JOIN pictures p ON p.id = j.picture_id
                 ORDER by
-                    pictures_to_process.nb_errors,
-                    pictures_to_process.ts
-                FOR UPDATE of pictures_to_process SKIP LOCKED
-                LIMIT 1
-            """
+                    j.nb_errors,
+                    j.ts
+                FOR UPDATE of j SKIP LOCKED
+                LIMIT 1"""
             ).fetchone()
             if r is None:
                 # Nothing to process
                 yield None
             else:
-                log.debug(f"Processing {r[0]}")
-
-                db_pic = DbPicture(id=str(r[0]), metadata=r[2])
-
-                with psycopg.connect(config["DB_URL"], autocommit=True) as reporting_conn:
-                    job = _initialize_picture_process(reporting_conn, pic=db_pic, task=ProcessTask(r[1]))
+                log.debug(f"Processing {r['id']}")
+
+                db_pic = DbPicture(id=r["picture_id"], metadata=r["metadata"]) if r["picture_id"] is not None else None
+                db_seq = DbSequence(id=r["sequence_id"]) if r["sequence_id"] is not None else None
+                db_upload_set = DbUploadSet(id=r["upload_set_id"]) if r["upload_set_id"] is not None else None
+
+                with app.pool.connection() as reporting_conn:
+                    job = _initialize_job(
+                        reporting_conn,
+                        job_queue_id=r["id"],
+                        db_pic=db_pic,
+                        db_seq=db_seq,
+                        db_upload_set=db_upload_set,
+                        task=ProcessTask(r["task"]),
+                    )
                     try:
                         yield job
 
                         # Finalize the picture process, set the picture status and remove the picture from the queue process
-                        _finalize_picture_process(locking_transaction, job)
-                        log.debug(f"Picture {db_pic.id} processed")
+                        _finalize_job(locking_transaction, job)
+                        log.debug(f"Job {job.label()} processed")
                     except RecoverableProcessException as e:
-                        _mark_process_as_error(locking_transaction, job, e, config, recoverable=True)
+                        _mark_process_as_error(locking_transaction, job, e, recoverable=True)
+                    except RetryLaterProcessException as e:
+                        _mark_process_as_error(locking_transaction, job, e, recoverable=True, mark_as_error=False)
                     except InterruptedError as interruption:
-                        log.error(f"Interruption received, stoping job {job.id} for picture {db_pic.id}")
+                        log.error(f"Interruption received, stoping job {job.label()}")
                         # starts a new connection, since the current one can be corrupted by the exception
-                        with psycopg.connect(config["DB_URL"], autocommit=True) as t:
-                            _mark_process_as_error(t, job, interruption, config, recoverable=True)
+                        with app.pool.connection() as t:
+                            _mark_process_as_error(t, job, interruption, recoverable=True)
                         error = interruption
                     except Exception as e:
-                        log.exception(f"Impossible to finish job {job.id} for picture {db_pic.id}")
-                        _mark_process_as_error(locking_transaction, job, e, config, recoverable=False)
+                        log.exception(f"Impossible to finish job {job.label()}")
+                        _mark_process_as_error(locking_transaction, job, e, recoverable=False)
 
                         # try to finalize the sequence anyway
-                        _finalize_sequence_if_last_picture(job)
+                        _finalize_sequence(job)
                         error = e
 
     # we raise an error after the transaction has been comited to be sure to have the state persisted in the database
@@ -318,163 +385,144 @@ def _get_next_picture_to_process(config):
         raise error
 
 
-def _finalize_sequence_if_last_picture(job: DbJob):
-    r = job.reporting_conn.execute(
-        """
-		SELECT sp.seq_id AS id FROM sequences_pictures AS sp
-		WHERE sp.pic_id = %(id)s
-	""",
-        {"id": job.pic.id},
-    ).fetchone()
-    if not r:
-        raise Exception(f"impossible to find sequence associated to picture {job.pic.id}")
-
-    seqId = r[0]
-
-    is_sequence_finalized = _is_sequence_finalized(job.reporting_conn, seqId)
-    if not is_sequence_finalized:
-        log.debug("sequence not finalized")
+def _finalize_sequence(job: DbJob):
+    # on picture preparation finalization, we add a sequence/upload_set finalization job
+    if job.task != "prepare" or not job.pic:
         return
 
-    with sentry_sdk.start_span(description="Finalizing sequence") as span:
-        span.set_data("sequence_id", seqId)
-        log.debug(f"Finalizing sequence {seqId}")
-
-        with utils.time.log_elapsed(f"Finalizing sequence {seqId}"):
-            # Complete missing headings in pictures
-            update_headings(job.reporting_conn, seqId)
-
-            # Change sequence database status in DB
-            # Also generates data in computed columns
-            job.reporting_conn.execute(
-                """WITH
-aggregated_pictures AS (
-    SELECT
-        sp.seq_id, 
-        MIN(p.ts::DATE) AS day,
-        ARRAY_AGG(DISTINCT TRIM(
-            CONCAT(p.metadata->>'make', ' ', p.metadata->>'model')
-        )) AS models,
-        ARRAY_AGG(DISTINCT p.metadata->>'type') AS types
-    FROM sequences_pictures sp
-    JOIN pictures p ON sp.pic_id = p.id
-    WHERE sp.seq_id = %(seq)s
-    GROUP BY sp.seq_id
-)
-UPDATE sequences
-SET
-    status = 'ready',
-    geom = compute_sequence_geom(id),
-    bbox = compute_sequence_bbox(id),
-    computed_type = CASE WHEN array_length(types, 1) = 1 THEN types[1] ELSE NULL END,
-    computed_model = CASE WHEN array_length(models, 1) = 1 THEN models[1] ELSE NULL END,
-    computed_capture_date = day
-FROM aggregated_pictures
-WHERE id = %(seq)s
-            """,
-                {"seq": seqId},
-            )
-
-            log.info(f"Sequence {seqId} is ready")
+    with job.reporting_conn.cursor(row_factory=dict_row) as cursor:
+        r = cursor.execute(
+            "SELECT upload_set_id, seq_id FROM pictures p LEFT JOIN sequences_pictures sp on sp.pic_id = p.id WHERE p.id = %(pic_id)s",
+            {"pic_id": job.pic.id},
+        ).fetchone()
 
+        if not r or not r["seq_id"]:
+            # if the associated upload set has not yet been dispatch, the picture might not be associated to a sequence
+            return
 
-def _is_sequence_finalized(db, seq_id: str):
-    """
-    We consider a sequence as ready, if all pictures have been processed and there is at least one correctly processed picture
-    Eg. we don't want pictures with preparing_status = 'not-processed' and at least one 'prepared'
-    """
-    statuses = db.execute(
-        """SELECT DISTINCT(preparing_status) FROM pictures p 
-        JOIN sequences_pictures sp ON sp.pic_id = p.id
-        WHERE
-            sp.seq_id = %(id)s
-            AND p.preparing_status <> 'broken'
-  		;
-	""",
-        {"id": seq_id},
-    ).fetchall()
+        if r["upload_set_id"]:
+            # if the picture is part of the upload set, the sequence finalization will be done when the upload set is dispatched
+            return
 
-    return [("prepared",)] == statuses
+        # Add a task to finalize the sequence/upload_set
+        sequences.add_finalization_job(cursor, r["seq_id"])
 
 
-def _finalize_picture_process(db, job: DbJob):
+def _finalize_job(conn, job: DbJob):
     job.reporting_conn.execute(
         "UPDATE job_history SET finished_at = CURRENT_TIMESTAMP WHERE id = %(id)s",
-        {"id": job.id},
+        {"id": job.job_history_id},
     )
-    if job.task == ProcessTask.prepare:
+    if job.task == ProcessTask.prepare and job.pic:
         # Note: the status is slowly been deprecated by replacing it with more precise status, and in the end it will be removed
         job.reporting_conn.execute(
             "UPDATE pictures SET status = 'ready', preparing_status = 'prepared' WHERE id = %(pic_id)s",
             {"pic_id": job.pic.id},
         )
 
-        # Check if we need to finalize the sequence
-        _finalize_sequence_if_last_picture(job)
-    elif job.task == ProcessTask.delete:
-        # TODO set the status to 'deleted' instead of removing it
-        db.execute(
+        # Add a task to finalize the sequence
+        _finalize_sequence(job)
+    elif job.task == ProcessTask.delete and job.pic:
+        conn.execute(
             "DELETE FROM pictures WHERE id = %(pic_id)s",
             {"pic_id": job.pic.id},
         )
-    db.execute(
-        "DELETE FROM pictures_to_process WHERE picture_id = %(pic_id)s",
-        {"pic_id": job.pic.id},
-    )
+    elif job.task == ProcessTask.delete and job.upload_set:
+        conn.execute(SQL("DELETE FROM upload_sets WHERE id = %s"), [job.upload_set.id])
+
+    conn.execute("DELETE FROM job_queue WHERE id = %(job_id)s", {"job_id": job.job_queue_id})
 
 
-def _initialize_picture_process(reporting_conn: psycopg.Connection, pic: DbPicture, task: ProcessTask) -> DbJob:
+def _initialize_job(
+    reporting_conn: psycopg.Connection,
+    job_queue_id: UUID,
+    db_pic: Optional[DbPicture],
+    db_seq: Optional[DbSequence],
+    db_upload_set: Optional[DbUploadSet],
+    task: ProcessTask,
+) -> DbJob:
     r = reporting_conn.execute(
-        """INSERT INTO job_history(picture_id, task)
-    VALUES (%(id)s, %(task)s)
-    RETURNING id
-	""",
-        {"id": pic.id, "task": task.value},
+        """INSERT INTO job_history(job_id, picture_id, sequence_id, upload_set_id, job_task)
+    VALUES (%(job_id)s, %(pic_id)s, %(seq_id)s, %(us_id)s, %(task)s)
+    RETURNING id""",
+        {
+            "job_id": job_queue_id,
+            "pic_id": db_pic.id if db_pic else None,
+            "seq_id": db_seq.id if db_seq else None,
+            "us_id": db_upload_set.id if db_upload_set else None,
+            "task": task.value,
+        },
     ).fetchone()
 
     if not r:
         raise Exception("impossible to insert task in database")
-    return DbJob(reporting_conn=reporting_conn, pic=pic, id=r[0], task=task)
+
+    return DbJob(
+        reporting_conn=reporting_conn,
+        job_queue_id=job_queue_id,
+        pic=db_pic,
+        seq=db_seq,
+        upload_set=db_upload_set,
+        task=task,
+        job_history_id=r[0],
+    )
 
 
-def _mark_process_as_error(db, job: DbJob, e: Exception, config, recoverable: bool = False):
+def _mark_process_as_error(conn, job: DbJob, e: Exception, recoverable: bool = False, mark_as_error: bool = True):
     job.reporting_conn.execute(
         """UPDATE job_history SET
 			error = %(err)s, finished_at = CURRENT_TIMESTAMP
 		WHERE id = %(id)s""",
-        {"err": str(e), "id": job.id},
+        {"err": str(e), "id": job.job_history_id},
     )
     if recoverable:
-        nb_error = db.execute(
-            """
-			UPDATE pictures_to_process SET
-				nb_errors = nb_errors + 1
-			WHERE picture_id = %(id)s
-            RETURNING nb_errors""",
-            {"err": str(e), "id": job.pic.id},
-        ).fetchone()
-        if nb_error and nb_error[0] > PICTURE_PROCESS_MAX_RETRY:
-            logging.info(f"Job to process picture {job.pic.id} has failed {nb_error} times, we stop trying to process it.")
-            recoverable = False
+        if mark_as_error:
+            nb_error = conn.execute(
+                """
+                UPDATE job_queue SET
+                    nb_errors = nb_errors + 1
+                WHERE id = %(id)s
+                RETURNING nb_errors""",
+                {"err": str(e), "id": job.job_queue_id},
+            ).fetchone()
+            if nb_error and nb_error[0] > PICTURE_PROCESS_MAX_RETRY:
+                logging.info(f"Job {job.label()} has failed {nb_error} times, we stop trying to process it.")
+                recoverable = False
+        else:
+            # it's not a real error, we just want to retry later
+            conn.execute(
+                SQL("UPDATE job_queue SET ts = NOW() WHERE id = %(id)s"),
+                {"err": str(e), "id": job.job_queue_id},
+            )
 
     if not recoverable:
         # Note: the status is slowly been deprecated by replacing it with more precise status, and in the end it will be removed
-        job.reporting_conn.execute(
-            """UPDATE pictures SET
-                preparing_status = 'broken', status = 'broken'
-            WHERE id = %(id)s""",
-            {"id": job.pic.id},
-        )
-        # on unrecoverable error, we remove the picture from the queue to process
-        db.execute(
-            """
-			DELETE FROM pictures_to_process
-				WHERE picture_id = %(id)s""",
-            {"id": job.pic.id},
-        )
+        if job.task == "prepare" and job.pic:
+            job.reporting_conn.execute(
+                """UPDATE pictures SET
+                    preparing_status = 'broken', status = 'broken'
+                WHERE id = %(id)s""",
+                {"id": job.pic.id},
+            )
+        # on unrecoverable error, we remove the job from the queue
+        conn.execute("DELETE FROM job_queue WHERE id = %(id)s", {"id": job.job_queue_id})
 
 
-def _delete_picture(job: DbJob):
+def _delete_picture(pic: DbPicture):
     """Delete a picture from the filesystem"""
-    log.debug(f"Deleting picture files {job.pic.id}")
-    utils.pictures.removeAllFiles(job.pic.id)
+    log.debug(f"Deleting picture files {pic.id}")
+    utils.pictures.removeAllFiles(pic.id)
+
+
+def _delete_upload_set(upload_set: DbUploadSet):
+    """Delete an upload set
+    We do this in the job queue since we want to wait for all its pictures to be deleted
+    """
+    with db.conn(current_app) as conn:
+        with conn.transaction(), conn.cursor() as cursor:
+            # we want to wait for all pictures to be deleted
+            has_more_pictures = cursor.execute("SELECT 1 FROM pictures WHERE upload_set_id = %s LIMIT 1", [upload_set.id]).fetchone()
+            if has_more_pictures and has_more_pictures[0]:
+                logging.info(f"More pictures to be deleted, upload_set {upload_set.id} will be deleted later")
+                raise RetryLaterProcessException("More pictures to be deleted, upload_set will be deleted later")
+            # Note: the real deletion will be done on job completion so the lock is released