geovisio 2.11.0__py3-none-any.whl → 2.12.0__py3-none-any.whl

geovisio/migrations/20240409_01_jnhra-pictures-grid.sql

@@ -0,0 +1,21 @@
+-- pictures-grid
+-- depends: 20240223_01_LsMHB-remove-binary-fields  20240308_01_aF0Jb-migrate-sequence-geom-multi-linestring
+
+-- it's mandatory for a vaccum analyze to be done to be able to use `ST_EstimatedExtent`
+
+CREATE MATERIALIZED VIEW pictures_grid AS
+SELECT
+    row_number() over () as id,
+    count(*) as nb_pictures,
+    g.geom
+FROM
+    ST_SquareGrid(
+        0.1,
+        ST_SetSRID(ST_EstimatedExtent('public', 'pictures', 'geom'), 4326)
+    ) AS g
+JOIN pictures AS p ON p.geom && g.geom
+JOIN sequences_pictures sp on sp.pic_id = p.id -- Added otherwise can't init from empty database
+GROUP BY g.geom;
+
+CREATE INDEX pictures_grid_id_idx ON pictures_grid(id);
+CREATE INDEX pictures_grid_geom_idx ON pictures_grid USING GIST(geom);

geovisio/migrations/20240416_01_FpyGs-pictures-stats-on-sequences.rollback.sql

@@ -0,0 +1,7 @@
+-- pictures_stats_on_sequences
+-- depends: 20240223_01_LsMHB-remove-binary-fields  20240308_01_aF0Jb-migrate-sequence-geom-multi-linestring
+
+ALTER TABLE sequences DROP COLUMN min_picture_ts;
+ALTER TABLE sequences DROP COLUMN max_picture_ts;
+ALTER TABLE sequences DROP COLUMN nb_pictures; 
+

geovisio/migrations/20240416_01_FpyGs-pictures-stats-on-sequences.sql

@@ -0,0 +1,9 @@
+-- pictures_stats_on_sequences
+-- depends: 20240223_01_LsMHB-remove-binary-fields  20240308_01_aF0Jb-migrate-sequence-geom-multi-linestring
+
+-- To avoid a costly join on sequence_pictures and picture, we add aggregated values at the sequence level, computed using triggers (in the next migration to split the table migration in 2 transactions)
+
+ALTER TABLE sequences
+ADD COLUMN min_picture_ts TIMESTAMPTZ,
+ADD COLUMN max_picture_ts TIMESTAMPTZ,
+ADD COLUMN nb_pictures BIGINT DEFAULT 0;

geovisio/migrations/20240416_02_A5KzC-fill-pictures-stats-on-sequences.rollback.sql

@@ -0,0 +1,5 @@
+-- fill_pictures_stats_on_sequences
+-- depends: 20240416_01_FpyGs-pictures-stats-on-sequences
+
+DROP TRIGGER picture_insertion_update_sequence_trigger ON pictures CASCADE;
+DROP TRIGGER seq_picture_deletion_update_sequence_trigger ON sequences_pictures CASCADE;

geovisio/migrations/20240416_02_A5KzC-fill-pictures-stats-on-sequences.sql

@@ -0,0 +1,84 @@
+-- fill_pictures_stats_on_sequences
+-- depends: 20240416_01_FpyGs-pictures-stats-on-sequences
+
+-- disable triggers on the current transaction, to avoid activating from update triggers only for a populating query
+-- (for example we don't want a sequence update_ts to be updated)
+SET session_replication_role = replica;
+
+WITH aggregated_pics AS (
+	SELECT sp.seq_id AS seq_id,
+		MIN(p.ts) AS min_picture_ts,
+		MAX(p.ts) AS max_picture_ts,
+		COUNT(p.*) AS nb_pictures
+	FROM sequences_pictures sp
+	JOIN pictures p ON sp.pic_id = p.id
+	GROUP BY sp.seq_id
+)
+UPDATE sequences SET
+	min_picture_ts = a.min_picture_ts,
+	max_picture_ts = a.max_picture_ts,
+	nb_pictures = a.nb_pictures
+FROM aggregated_pics a
+WHERE sequences.id = seq_id;
+
+-- put back the triggers
+SET session_replication_role = DEFAULT;
+
+-- Also add Triggers to maintain those counters
+
+-- On picture insertion (usually do nothing, we want pictures to be ready to count them)
+CREATE OR REPLACE FUNCTION update_sequence_on_picture_insertion() RETURNS TRIGGER AS
+$BODY$
+BEGIN
+	UPDATE sequences
+	SET 
+		nb_pictures = nb_pictures + 1,
+		min_picture_ts = LEAST(min_picture_ts, NEW.ts),
+		max_picture_ts = GREATEST(max_picture_ts, NEW.ts)
+	WHERE id IN (
+		SELECT DISTINCT(seq_id) FROM sequences_pictures WHERE pic_id = NEW.id
+	);
+
+    RETURN NULL;
+END;
+$BODY$
+language plpgsql;
+
+CREATE CONSTRAINT TRIGGER picture_insertion_update_sequence_trigger
+    AFTER INSERT ON pictures
+    DEFERRABLE INITIALLY DEFERRED -- check at end of transaction for picture to be linked to a sequence
+    FOR EACH ROW
+    EXECUTE PROCEDURE update_sequence_on_picture_insertion();
+
+-- On picture deletion (don't try to do clever stuff, compute from scratch)
+CREATE OR REPLACE FUNCTION update_sequence_on_sequences_pictures_deletion() RETURNS TRIGGER AS
+$BODY$
+BEGIN
+	WITH aggregated_pics AS (
+		SELECT sp.seq_id AS seq_id,
+			MIN(p.ts) AS min_picture_ts,
+			MAX(p.ts) AS max_picture_ts,
+			COUNT(p.*) AS nb_pictures
+		FROM sequences_pictures sp
+		JOIN pictures p ON sp.pic_id = p.id
+		WHERE sp.seq_id IN (
+				SELECT DISTINCT(seq_id) FROM old_table
+			)
+		GROUP BY sp.seq_id
+	)
+	UPDATE sequences SET
+		min_picture_ts = a.min_picture_ts,
+		max_picture_ts = a.max_picture_ts,
+		nb_pictures = a.nb_pictures
+	FROM aggregated_pics a
+	WHERE sequences.id = seq_id;
+    RETURN NULL;
+END;
+$BODY$
+language plpgsql;
+
+CREATE TRIGGER seq_picture_deletion_update_sequence_trigger
+    AFTER DELETE ON sequences_pictures
+	REFERENCING OLD TABLE AS old_table
+    FOR EACH STATEMENT -- run this for each statement
+    EXECUTE PROCEDURE update_sequence_on_sequences_pictures_deletion();

geovisio/migrations/20240507_01_eBfqZ-refresh-table.rollback.sql

@@ -0,0 +1,5 @@
+-- refresh_table
+-- depends: 20240409_01_jnhra-pictures-grid  20240416_02_A5KzC-fill-pictures-stats-on-sequences
+
+
+DROP TABLE refresh_database;

geovisio/migrations/20240507_01_eBfqZ-refresh-table.sql

@@ -0,0 +1,9 @@
+-- refresh_table
+-- depends: 20240409_01_jnhra-pictures-grid  20240416_02_A5KzC-fill-pictures-stats-on-sequences
+
+
+CREATE TABLE refresh_database(
+	refreshed_at TIMESTAMPTZ
+);
+
+INSERT INTO refresh_database(refreshed_at) VALUES (null);

geovisio/migrations/20240507_02_dzVET-picture-grid-public.rollback.sql

@@ -0,0 +1,21 @@
+-- picture_grid_public
+-- depends: 20240507_01_eBfqZ-refresh-table
+
+DROP MATERIALIZED VIEW pictures_grid;
+
+CREATE MATERIALIZED VIEW pictures_grid AS
+SELECT
+    row_number() over () as id,
+    count(*) as nb_pictures,
+    g.geom
+FROM
+    ST_SquareGrid(
+        0.1,
+        ST_SetSRID(ST_EstimatedExtent('public', 'pictures', 'geom'), 4326)
+    ) AS g
+JOIN pictures AS p ON p.geom && g.geom
+JOIN sequences_pictures sp on sp.pic_id = p.id
+GROUP BY g.geom;
+
+CREATE INDEX pictures_grid_id_idx ON pictures_grid(id);
+CREATE INDEX pictures_grid_geom_idx ON pictures_grid USING GIST(geom);

geovisio/migrations/20240507_02_dzVET-picture-grid-public.sql

@@ -0,0 +1,26 @@
+-- picture_grid_public
+-- depends: 20240507_01_eBfqZ-refresh-table
+-- transactional: false
+
+BEGIN;
+
+DROP MATERIALIZED VIEW pictures_grid;
+
+--only consider public pictures in this view
+CREATE MATERIALIZED VIEW pictures_grid AS
+SELECT
+    row_number() over () as id,
+    count(*) as nb_pictures,
+    g.geom
+FROM
+    ST_SquareGrid(
+        0.1,
+        ST_SetSRID(ST_EstimatedExtent('public', 'pictures', 'geom'), 4326)
+    ) AS g
+JOIN pictures AS p ON p.geom && g.geom
+JOIN sequences_pictures sp on sp.pic_id = p.id
+JOIN sequences s ON s.id = sp.seq_id
+WHERE p.status = 'ready' and s.status = 'ready'
+GROUP BY g.geom;
+
+COMMIT;

geovisio/migrations/20240514_01_IT7DD-picture-delete-cascade.rollback.sql

@@ -0,0 +1,41 @@
+-- picture_delete_cascade
+-- depends: 20240507_02_dzVET-picture-grid-public
+
+
+-- only recompute sequences shape for non deleted sequences
+CREATE OR REPLACE FUNCTION sequences_pictures_delete() RETURNS trigger AS $$
+BEGIN
+	UPDATE sequences
+	SET
+		geom = compute_sequence_geom(id),
+		bbox = compute_sequence_bbox(id)
+	WHERE id IN (SELECT DISTINCT seq_id FROM old_table);
+
+	RETURN NULL;
+END $$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION update_sequence_on_sequences_pictures_deletion() RETURNS TRIGGER AS
+$BODY$
+BEGIN
+	WITH aggregated_pics AS (
+		SELECT sp.seq_id AS seq_id,
+			MIN(p.ts) AS min_picture_ts,
+			MAX(p.ts) AS max_picture_ts,
+			COUNT(p.*) AS nb_pictures
+		FROM sequences_pictures sp
+		JOIN pictures p ON sp.pic_id = p.id
+		WHERE sp.seq_id IN (
+				SELECT DISTINCT(seq_id) FROM old_table
+			)
+		GROUP BY sp.seq_id
+	)
+	UPDATE sequences SET
+		min_picture_ts = a.min_picture_ts,
+		max_picture_ts = a.max_picture_ts,
+		nb_pictures = a.nb_pictures
+	FROM aggregated_pics a
+	WHERE sequences.id = seq_id;
+    RETURN NULL;
+END;
+$BODY$
+language plpgsql;

geovisio/migrations/20240514_01_IT7DD-picture-delete-cascade.sql

@@ -0,0 +1,42 @@
+-- picture_delete_cascade
+-- depends: 20240507_02_dzVET-picture-grid-public
+
+
+-- only recompute sequences shape for non deleted sequences
+CREATE OR REPLACE FUNCTION sequences_pictures_delete() RETURNS trigger AS $$
+BEGIN
+	UPDATE sequences
+	SET
+		geom = compute_sequence_geom(id),
+		bbox = compute_sequence_bbox(id)
+	WHERE id IN (SELECT DISTINCT seq_id FROM old_table) AND status <> 'deleted';
+
+	RETURN NULL;
+END $$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION update_sequence_on_sequences_pictures_deletion() RETURNS TRIGGER AS
+$BODY$
+BEGIN
+	WITH aggregated_pics AS (
+		SELECT sp.seq_id AS seq_id,
+			MIN(p.ts) AS min_picture_ts,
+			MAX(p.ts) AS max_picture_ts,
+			COUNT(p.*) AS nb_pictures
+		FROM sequences_pictures sp
+		JOIN pictures p ON sp.pic_id = p.id
+		JOIN sequences s ON sp.seq_id = s.id
+		WHERE sp.seq_id IN (
+				SELECT DISTINCT(seq_id) FROM old_table
+			) AND s.status <> 'deleted'
+		GROUP BY sp.seq_id
+	)
+	UPDATE sequences SET
+		min_picture_ts = a.min_picture_ts,
+		max_picture_ts = a.max_picture_ts,
+		nb_pictures = a.nb_pictures
+	FROM aggregated_pics a
+	WHERE sequences.id = seq_id;
+    RETURN NULL;
+END;
+$BODY$
+language plpgsql;

geovisio/migrations/20240611_01_jftHn-content-md5.rollback.sql

@@ -0,0 +1,5 @@
+-- content_md5
+-- depends: 20240514_01_IT7DD-picture-delete-cascade
+
+ALTER TABLE pictures DROP COLUMN IF EXISTS original_content_md5;
+

geovisio/migrations/20240611_01_jftHn-content-md5.sql

@@ -0,0 +1,7 @@
+-- content_md5
+-- depends: 20240514_01_IT7DD-picture-delete-cascade
+
+-- concurrent index cannot be run inside a transaction
+-- transactional: false
+
+ALTER TABLE pictures ADD COLUMN IF NOT EXISTS original_content_md5 UUID;

geovisio/migrations/20240612_01_yNcuE-upload-set.rollback.sql

@@ -0,0 +1,6 @@
+-- upload_set
+-- depends: 20240514_01_IT7DD-picture-delete-cascade
+
+ALTER TABLE pictures DROP COLUMN IF EXISTS upload_set_id;
+DROP TABLE upload_sets;
+DROP TYPE upload_set_sort_method;

geovisio/migrations/20240612_01_yNcuE-upload-set.sql

@@ -0,0 +1,46 @@
+-- upload_set
+-- depends: 20240514_01_IT7DD-picture-delete-cascade
+
+--transactional: false
+
+BEGIN;
+
+CREATE TYPE upload_set_sort_method AS ENUM (
+	'filename_asc',
+	'filename_desc',
+	'time_asc',
+	'time_desc'
+);
+
+
+CREATE TABLE upload_sets(
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    account_id UUID NOT NULL REFERENCES accounts(id) ON DELETE CASCADE,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    completed BOOLEAN NOT NULL DEFAULT false,
+    dispatched BOOLEAN NOT NULL DEFAULT false,
+    estimated_nb_files INT,
+
+    sort_method upload_set_sort_method NOT NULL DEFAULT 'time_asc',
+    split_time INTERVAL DEFAULT INTERVAL '1 minute',
+    duplicate_distance real DEFAULT 1,
+    split_distance INT DEFAULT 100,
+    duplicate_rotation INT DEFAULT 30,
+
+    title VARCHAR NOT NULL
+);
+
+COMMENT ON COLUMN upload_sets.sort_method IS 'Strategy used for sorting your pictures. Either by filename or EXIF time, in ascending or descending order.';
+COMMENT ON COLUMN upload_sets.split_distance IS 'Maximum distance between two pictures to be considered in the same sequence (in meters).';
+COMMENT ON COLUMN upload_sets.split_time IS 'Maximum time interval between two pictures to be considered in the same sequence.';
+COMMENT ON COLUMN upload_sets.duplicate_distance IS 'Maximum distance between two pictures to be considered as duplicates (in meters).';
+COMMENT ON COLUMN upload_sets.duplicate_rotation IS 'Maximum angle of rotation for two too-close-pictures to be considered as duplicates (in degrees).';
+
+ALTER TABLE pictures ADD COLUMN IF NOT EXISTS upload_set_id UUID;
+
+ALTER TABLE pictures ADD CONSTRAINT upload_set_fk_id FOREIGN KEY (upload_set_id) REFERENCES upload_sets(id) NOT VALID;
+
+COMMIT;
+
+-- The foreign kye constraint is added as invalid first, then validated, since it will allow row updating while validating the constraint
+ALTER TABLE pictures VALIDATE CONSTRAINT upload_set_fk_id;

geovisio/migrations/20240617_01_tKtlx-md5-concurrent-index.rollback.sql

@@ -0,0 +1,4 @@
+-- md5_concurrent_index
+-- depends: 20240611_01_jftHn-content-md5
+
+DROP INDEX IF EXISTS pictures_original_content_md5_idx;

geovisio/migrations/20240617_01_tKtlx-md5-concurrent-index.sql

@@ -0,0 +1,8 @@
+-- md5_concurrent_index
+-- depends: 20240611_01_jftHn-content-md5
+
+-- transactional: false
+
+
+CREATE INDEX CONCURRENTLY pictures_original_content_md5_idx ON pictures(original_content_md5);
+

geovisio/migrations/20240625_01_XMZ24-fix-sequence-stat-on-pic-insertion.rollback.sql

@@ -0,0 +1,30 @@
+-- fix_sequence_stat_on_pic_insertion
+-- depends: 20240612_01_yNcuE-upload-set  20240617_01_tKtlx-md5-concurrent-index
+
+-- put back old triggers
+DROP TRIGGER sequences_pictures_insertion_update_sequence_trigger ON sequences_pictures CASCADE;
+
+-- DROP TRIGGER picture_insertion_update_sequence_trigger ON pictures CASCADE;
+CREATE OR REPLACE FUNCTION update_sequence_on_picture_insertion() RETURNS TRIGGER AS
+$BODY$
+BEGIN
+	UPDATE sequences
+	SET 
+		nb_pictures = nb_pictures + 1,
+		min_picture_ts = LEAST(min_picture_ts, NEW.ts),
+		max_picture_ts = GREATEST(max_picture_ts, NEW.ts)
+	WHERE id IN (
+		SELECT DISTINCT(seq_id) FROM sequences_pictures WHERE pic_id = NEW.id
+	);
+
+    RETURN NULL;
+END;
+$BODY$
+language plpgsql;
+
+CREATE CONSTRAINT TRIGGER picture_insertion_update_sequence_trigger
+    AFTER INSERT ON pictures
+    DEFERRABLE INITIALLY DEFERRED -- check at end of transaction for picture to be linked to a sequence
+    FOR EACH ROW
+    EXECUTE PROCEDURE update_sequence_on_picture_insertion();
+

geovisio/migrations/20240625_01_XMZ24-fix-sequence-stat-on-pic-insertion.sql

@@ -0,0 +1,31 @@
+-- fix_sequence_stat_on_pic_insertion
+-- depends: 20240612_01_yNcuE-upload-set  20240617_01_tKtlx-md5-concurrent-index
+
+-- drop old triggers on insertion on pictures table, we want the trigger to be on sequences_pictures instead
+
+DROP TRIGGER picture_insertion_update_sequence_trigger ON pictures CASCADE;
+
+CREATE OR REPLACE FUNCTION update_sequence_on_picture_insertion() RETURNS TRIGGER AS
+$BODY$
+BEGIN
+    WITH p AS (
+        SELECT NEW.seq_id, ts FROM pictures WHERE id = NEW.pic_id
+    )
+	UPDATE sequences
+	SET
+		nb_pictures = nb_pictures + 1,
+		min_picture_ts = LEAST(min_picture_ts, p.ts),
+		max_picture_ts = GREATEST(max_picture_ts, p.ts)
+    FROM p
+	WHERE id = p.seq_id;
+
+    RETURN NULL;
+END;
+$BODY$
+language plpgsql;
+
+CREATE CONSTRAINT TRIGGER sequences_pictures_insertion_update_sequence_trigger
+    AFTER INSERT ON sequences_pictures
+    DEFERRABLE INITIALLY DEFERRED -- check at end of transaction for picture to be linked to a sequence
+    FOR EACH ROW
+    EXECUTE PROCEDURE update_sequence_on_picture_insertion();

geovisio/migrations/20240708_01_Xn7IH-job-queue.rollback.sql

@@ -0,0 +1,31 @@
+-- job_queue
+-- depends: 20240625_01_XMZ24-fix-sequence-stat-on-pic-insertion
+
+ALTER TABLE job_history DROP COLUMN job_id;
+ALTER TABLE job_history DROP COLUMN upload_set_id;
+ALTER TABLE job_history DROP COLUMN sequence_id;
+ALTER TABLE job_history DROP COLUMN job_task;
+
+DELETE FROM job_history WHERE picture_id IS NULL; -- remove new style job history
+ALTER TABLE job_history ALTER COLUMN picture_id SET NOT NULL;
+
+DROP VIEW pictures_to_process;
+DROP TABLE job_queue;
+DROP TYPE job_type;
+
+
+-- put back old table
+CREATE TABLE pictures_to_process(
+	picture_id UUID PRIMARY KEY,
+	ts TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    task picture_process_task DEFAULT 'prepare',
+    nb_errors INT NOT NULL DEFAULT 0
+);
+ALTER TABLE pictures_to_process ADD CONSTRAINT picture_id_fk FOREIGN KEY (picture_id) REFERENCES pictures (id);
+
+CREATE INDEX pictures_to_process_nb_errors_ts_idx ON pictures_to_process(nb_errors, ts);
+
+CREATE INDEX pictures_to_process_ts_idx ON pictures_to_process(ts);
+
+DROP TRIGGER upload_sets_completion_trg ON upload_sets;
+DROP COLLATION numeric_file_names;

geovisio/migrations/20240708_01_Xn7IH-job-queue.sql

@@ -0,0 +1,104 @@
+-- job_queue
+-- depends: 20240625_01_XMZ24-fix-sequence-stat-on-pic-insertion
+
+CREATE TYPE job_type AS ENUM (
+	'prepare', -- picture needs to be prepared to be published
+	'delete', -- picture needs to be deleted
+	'dispatch', -- upload set needs to be dispatched into several collections
+	'finalize' -- finalize a collection
+);
+
+CREATE TABLE job_queue (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+
+    -- job can be linked to either a picture, an upload set or a sequence
+    picture_id UUID REFERENCES pictures(id) ON DELETE CASCADE,
+    upload_set_id UUID REFERENCES upload_sets(id) ON DELETE CASCADE,
+    sequence_id UUID REFERENCES sequences(id) ON DELETE CASCADE,
+
+	ts TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    
+    -- If not null, this job will not be done before this timestamp
+	to_do_after_ts TIMESTAMPTZ,
+
+    nb_errors INT NOT NULL DEFAULT 0,
+
+    task job_type NOT NULL,
+
+	CONSTRAINT picture_id_unique UNIQUE (picture_id),
+	CONSTRAINT upload_set_id_unique UNIQUE (upload_set_id),
+	CONSTRAINT sequence_id_unique UNIQUE (sequence_id) 
+);
+
+-- Add a constraint to ensure that one and only one of the three foreign keys is set
+ALTER TABLE job_queue
+ADD CONSTRAINT one_external_id CHECK (num_nonnulls(picture_id, upload_set_id, sequence_id) = 1);
+
+CREATE INDEX job_queue_nb_errors_ts_idx ON job_queue(nb_errors, ts);
+
+-- this index is used to handle a 'ON CONFLICT' clause in the collection delete
+CREATE INDEX job_queue_picture_id_idx ON job_queue(picture_id);
+
+INSERT INTO job_queue (picture_id, ts, task, nb_errors)
+SELECT 
+    picture_id, 
+    ts, 
+    case WHEN task = 'delete' THEN 'delete'::job_type ELSE 'prepare'::job_type END, 
+    nb_errors
+FROM pictures_to_process;
+
+DROP TABLE pictures_to_process CASCADE;
+
+CREATE VIEW pictures_to_process AS (
+    SELECT picture_id, ts, task, nb_errors FROM job_queue WHERE task IN ('prepare', 'delete')
+);
+
+-- we change the trigger that insert all new pictures into the pictures_to_process table
+CREATE
+OR REPLACE FUNCTION picture_insertion() RETURNS TRIGGER AS $BODY$ BEGIN
+	INSERT INTO
+		job_queue(picture_id, task)
+	VALUES
+		(NEW.id, 'prepare');
+    RETURN NEW;
+END;
+$BODY$ LANGUAGE plpgsql;
+
+-- Link the job_history table to the new job_queue table
+
+ALTER TABLE job_history ADD COLUMN job_id UUID; -- do not add a foreign key constraint yet since the job is only temporarily in the job_queue and cannot add non nullity for old history
+ALTER TABLE job_history ADD COLUMN upload_set_id UUID REFERENCES upload_sets(id) ON DELETE CASCADE;
+ALTER TABLE job_history ADD COLUMN sequence_id UUID REFERENCES sequences(id) ON DELETE CASCADE;
+
+ALTER TABLE job_history ADD COLUMN job_task job_type NOT NULL DEFAULT 'prepare';
+ALTER TABLE job_history ALTER COLUMN picture_id DROP NOT NULL;
+ALTER TABLE job_history
+ADD CONSTRAINT one_external_id CHECK (num_nonnulls(picture_id, upload_set_id, sequence_id) = 1);
+
+UPDATE job_history SET job_task = case WHEN task = 'delete' THEN 'delete'::job_type ELSE 'prepare'::job_type END where job_task IS NULL;
+
+
+CREATE INDEX job_history_upload_set_id_idx ON job_history(upload_set_id);
+CREATE INDEX job_history_sequence_id_idx ON job_history(sequence_id);
+
+-- add a trigger on upload_set completion to dispatch it
+CREATE
+OR REPLACE FUNCTION upload_set_completion() RETURNS TRIGGER AS $BODY$ BEGIN
+
+    IF NEW.completed THEN
+        INSERT INTO
+            job_queue(upload_set_id, task)
+        VALUES
+            (NEW.id, 'dispatch')
+        ON CONFLICT (upload_set_id) DO UPDATE SET ts = CURRENT_TIMESTAMP;
+    END IF;
+    RETURN NEW;
+END;
+$BODY$ LANGUAGE plpgsql;
+
+CREATE TRIGGER upload_sets_completion_trg
+AFTER UPDATE OF completed ON upload_sets
+FOR EACH ROW EXECUTE FUNCTION upload_set_completion();
+
+-- Create a collation to sort file names using natural sort (with numbers mixed with letters)
+CREATE COLLATION numeric_file_names (provider = icu, locale = 'en@colNumeric=yes'); 

geovisio/migrations/20240715_01_Hca9V-upload-set-metadata.rollback.sql

@@ -0,0 +1,4 @@
+-- upload_set_metadata
+-- depends: 20240708_01_Xn7IH-job-queue
+
+ALTER TABLE upload_sets DROP COLUMN metadata;

geovisio/migrations/20240715_01_Hca9V-upload-set-metadata.sql

@@ -0,0 +1,5 @@
+-- upload_set_metadata
+-- depends: 20240708_01_Xn7IH-job-queue
+
+ALTER TABLE upload_sets
+  ADD COLUMN metadata jsonb;

geovisio/migrations/20240723_01_ePGFe-upload-set-files.rollback.sql

@@ -0,0 +1,7 @@
+-- upload_set files
+-- depends: 20240715_01_Hca9V-upload-set-metadata
+
+DROP TABLE files;
+DROP TYPE file_type;
+
+DROP INDEX pictures_upload_set_idx;

geovisio/migrations/20240723_01_ePGFe-upload-set-files.sql

@@ -0,0 +1,29 @@
+-- upload_set files
+-- depends: 20240715_01_Hca9V-upload-set-metadata
+
+-- transactional: false
+
+BEGIN;
+-- Note: for the moment we only support pictures, but later we might accept more kind of files (like gpx traces, video, ...)
+CREATE TYPE file_type AS ENUM (
+	'picture'
+);
+
+CREATE TABLE files(
+    upload_set_id UUID NOT NULL REFERENCES upload_sets(id) ON DELETE CASCADE,
+    picture_id UUID REFERENCES pictures(id) ON DELETE CASCADE, -- some files might not be associated to a picture
+    inserted_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    file_type file_type NOT NULL,
+    size INT NOT NULL,
+    file_name VARCHAR NOT NULL,
+    content_md5 UUID NOT NULL,
+    rejection_reason VARCHAR
+);
+
+-- We consider that the file_name should be unique for a given upload_set, this way we can detect if an upload is an update of an existing file
+CREATE UNIQUE INDEX files_upload_set_name_idx ON files(upload_set_id, file_name);
+
+COMMIT;
+
+-- also add an index on upload_set_id for the pictures table
+CREATE INDEX CONCURRENTLY pictures_upload_set_idx ON pictures(upload_set_id);

geovisio/migrations/20240729_01_HALjj-upload-set-sort.rollback.sql

@@ -0,0 +1,18 @@
+-- upload_set_sort
+-- depends: 20240723_01_ePGFe-upload-set-files
+CREATE TYPE upload_set_sort_method_new AS ENUM (
+    'filename_asc',
+    'filename_desc',
+    'time_asc',
+    'time_desc'
+);
+ALTER TABLE upload_sets
+ALTER COLUMN sort_method DROP DEFAULT;
+ALTER TABLE upload_sets
+ALTER COLUMN sort_method TYPE upload_set_sort_method_new USING sort_method::text::upload_set_sort_method_new;
+ALTER TABLE upload_sets
+ALTER COLUMN sort_method
+SET DEFAULT 'time_asc';
+DROP TYPE upload_set_sort_method;
+ALTER TYPE upload_set_sort_method_new
+RENAME TO upload_set_sort_method;

geovisio/migrations/20240729_01_HALjj-upload-set-sort.sql

@@ -0,0 +1,18 @@
+-- upload_set_sort
+-- depends: 20240723_01_ePGFe-upload-set-files
+CREATE TYPE upload_set_sort_method_new AS ENUM (
+    'filename-asc',
+    'filename-desc',
+    'time-asc',
+    'time-desc'
+);
+ALTER TABLE upload_sets
+ALTER COLUMN sort_method DROP DEFAULT;
+ALTER TABLE upload_sets
+ALTER COLUMN sort_method TYPE upload_set_sort_method_new USING sort_method::text::upload_set_sort_method_new;
+ALTER TABLE upload_sets
+ALTER COLUMN sort_method
+SET DEFAULT 'time-asc';
+DROP TYPE upload_set_sort_method;
+ALTER TYPE upload_set_sort_method_new
+RENAME TO upload_set_sort_method;