geovisio 2.11.0__py3-none-any.whl → 2.12.0__py3-none-any.whl

geovisio/migrations/20241011_01_e1j5C-pic-quality.sql

@@ -0,0 +1,249 @@
+-- pic_quality
+-- depends: 20240912_01_dAALm-account-index
+
+-- NOTE : this migration is split in two files
+--   this one, and next migration pic_quality_update
+--   this migration is transactional, the next one is not
+
+
+-- Add common cameras focal length (for FOV + HPixelDensity)
+-- NOTE : cameras / sensor_data is not used anymore in versions > 2.7.1
+--       For updating cameras metadata, please see GeoPicture Tag Reader repository
+INSERT INTO cameras(model, sensor_width) VALUES
+('samsung SM-A336B', 6.4),
+('Viofo A119 Mini 2', 5.2),
+('GoPro HERO9 Black', 6.17),
+('GoPro HERO5 Black', 6.17),
+('HUAWEI EML-L29', 6.29),
+('Panasonic DC-LX100M2', 17.3),
+('SONY FDR-X1000V', 6.19),
+('GoPro Max', 6.17),
+('samsung SM-G950F',7.05),
+('GoPro HERO7 Black',6.17),
+('Xiaomi M2101K6G',8.4),
+('HUAWEI VOG-L29',7.3),
+('samsung SM-A546B', 12),
+('GoPro HERO10 Black', 6.17),
+('Xiaomi 2107113SG',7.4),
+('Xiaomi Redmi Note 9 Pro',7.2),
+('GoPro HERO11 Black', 6.74),
+('OnePlus A5000', 5.22),
+('samsung SM-A500FU', 4.69),
+('XIAOYI YDXJ 2', 10.2),
+('Apple iPhone 6s', 4.8),
+('samsung SM-G901F', 5.95),
+('OnePlus A3003', 6.4),
+('GoPro HERO8 Black', 6.17),
+('XIAOYI YDXJ 1', 10.2);
+
+
+-- Complete FOV for imported pictures
+CREATE OR REPLACE FUNCTION missing_fov(metadata JSONB) RETURNS JSONB AS $$
+DECLARE
+	sensor_width FLOAT;
+BEGIN
+	IF metadata->>'field_of_view' IS NOT NULL OR metadata->>'focal_length' IS NULL OR (metadata->>'focal_length')::float = 0 THEN
+		RETURN metadata;
+	END IF;
+
+	-- Find appropriate sensor width
+	IF metadata->>'make' = 'samsung' AND metadata->>'model' = 'SM-A336B' THEN
+		sensor_width :=  6.4;
+	ELSIF metadata->>'make' = 'Viofo' AND metadata->>'model' = 'A119 Mini 2' THEN
+		sensor_width := 5.2;
+	ELSIF metadata->>'make' = 'GoPro' AND metadata->>'model' = 'HERO9 Black' THEN
+		sensor_width :=  6.17;
+	ELSIF metadata->>'make' = 'GoPro' AND metadata->>'model' = 'HERO5 Black' THEN
+		sensor_width :=  6.17;
+	ELSIF metadata->>'make' = 'HUAWEI' AND metadata->>'model' = 'EML-L29' THEN
+		sensor_width :=  6.29;
+	ELSIF metadata->>'make' = 'Panasonic' AND metadata->>'model' = 'DC-LX100M2' THEN
+		sensor_width :=  17.3;
+	ELSIF metadata->>'make' = 'SONY' AND metadata->>'model' = 'FDR-X1000V' THEN
+		sensor_width :=  6.19;
+	ELSIF metadata->>'make' = 'GoPro' AND metadata->>'model' = 'Max' THEN
+		sensor_width := 6.17;
+	ELSIF metadata->>'make' = 'samsung' AND metadata->>'model' = 'SM-G950F' THEN
+		sensor_width := 7.05;
+	ELSIF metadata->>'make' = 'GoPro' AND metadata->>'model' = 'HERO7 Black' THEN
+		sensor_width := 6.17;
+	ELSIF metadata->>'make' = 'Xiaomi' AND metadata->>'model' = 'M2101K6G' THEN
+		sensor_width := 8.4;
+	ELSIF metadata->>'make' = 'HUAWEI' AND metadata->>'model' = 'VOG-L29' THEN
+		sensor_width := 7.3;
+	ELSIF metadata->>'make' = 'samsung' AND metadata->>'model' = 'SM-A546B' THEN
+		sensor_width := 12;
+	ELSIF metadata->>'make' = 'GoPro' AND metadata->>'model' = 'HERO10 Black' THEN
+		sensor_width :=  6.17;
+	ELSIF metadata->>'make' = 'Xiaomi' AND metadata->>'model' = '2107113SG' THEN
+		sensor_width := 7.4;
+	ELSIF metadata->>'make' = 'Xiaomi' AND metadata->>'model' = 'Redmi Note 9 Pro' THEN
+		sensor_width := 7.2;
+	ELSIF metadata->>'make' = 'GoPro' AND metadata->>'model' = 'HERO11 Black' THEN
+		sensor_width := 6.74;
+	ELSIF metadata->>'make' = 'OnePlus' AND metadata->>'model' = 'A5000' THEN
+		sensor_width := 5.22;
+	ELSIF metadata->>'make' = 'samsung' AND metadata->>'model' = 'SM-A500FU' THEN
+		sensor_width := 4.69;
+	ELSIF metadata->>'make' = 'XIAOYI' AND metadata->>'model' = 'YDXJ 2' THEN
+		sensor_width := 10.2;
+	ELSIF metadata->>'make' = 'Apple' AND metadata->>'model' = 'iPhone 6s' THEN
+		sensor_width := 4.8;
+	ELSIF metadata->>'make' = 'samsung' AND metadata->>'model' = 'SM-G901F' THEN
+		sensor_width := 5.95;
+	ELSIF metadata->>'make' = 'OnePlus' AND metadata->>'model' = 'A3003' THEN
+		sensor_width := 6.4;
+	ELSIF metadata->>'make' = 'GoPro' AND metadata->>'model' = 'HERO8 Black' THEN
+		sensor_width := 6.17;
+	ELSIF metadata->>'make' = 'XIAOYI' AND metadata->>'model' = 'YDXJ 1' THEN
+		sensor_width := 10.2;
+	-- If not found, just send back original metadata
+	ELSE
+		RETURN metadata;
+	END IF;
+
+	-- Update metadata
+	RETURN jsonb_set(
+		metadata,
+		'{field_of_view}'::text[],
+		((ROUND(DEGREES(2 * ATAN(sensor_width / (2 * (metadata->>'focal_length')::float)))))::varchar)::jsonb
+	);
+END;
+$$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
+
+
+-- Read float values from EXIF
+CREATE OR REPLACE FUNCTION get_float(val VARCHAR) RETURNS FLOAT AS $$
+DECLARE
+	list VARCHAR[];
+BEGIN
+	IF val ~ '^\d+\/\d+$' THEN
+		list := regexp_split_to_array(val, '/');
+		RETURN list[1]::float / list[2]::float;
+    ELSIF val ~ '^\d+(\.\d+)?$' THEN
+        RETURN val::float;
+    ELSE
+        RETURN NULL;
+    END IF;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
+
+
+-- GPS accuracy function
+CREATE OR REPLACE FUNCTION gps_accuracy(metadata JSONB, exif JSONB) RETURNS FLOAT AS $$
+DECLARE
+	gps_dop FLOAT;
+	gps_diff INT;
+	gps_hpos_err FLOAT;
+BEGIN
+	-- Parse GPS DOP, either float or fraction
+	gps_dop := get_float(COALESCE(exif->>'Exif.GPSInfo.GPSDOP', exif->>'Xmp.exif.GPSDOP', ''));
+	gps_diff := (COALESCE(exif->>'Exif.GPSInfo.GPSDifferential', exif->>'Xmp.exif.GPSDifferential'))::INT;
+	gps_hpos_err := get_float(COALESCE(exif->>'Exif.GPSInfo.GPSHPositioningError', exif->>'Xmp.exif.GPSHPositioningError'));
+
+	-- Direct horizontal positioning error in meters -> return as is
+	IF gps_hpos_err IS NOT NULL AND gps_hpos_err > 0 THEN
+		RETURN gps_hpos_err;
+	
+	-- GPS DOP available
+	ELSIF gps_dop IS NOT NULL AND gps_dop > 0 THEN
+		-- With a DGPS -> consider GPS nominal error as 1 meter
+		IF gps_diff = 1 THEN
+			RETURN gps_dop;
+		
+		-- Without DGPS -> consider GPS nominal error as 3 meters in average
+		ELSE
+			RETURN 3 * gps_dop;
+		END IF;
+	
+	-- DGPS -> return 2 meters precision
+	ELSIF gps_diff = 1 THEN
+		RETURN 2;
+
+	-- Approximate guesses based on model
+	ELSIF metadata->>'make' IS NOT NULL OR metadata->>'model' IS NOT NULL THEN
+		-- Good non-diff GPS devices (best case is 3m, so setting 4 for tolerance)
+		IF lower(metadata->>'make') IN (
+			'gopro', 'insta360', 'garmin', 'viofo', 'xiaoyi', 'blackvue', 'tectectec',
+			'arashi vision'
+		)
+		OR metadata->>'model' IN ('LG-R105', 'FDR-X1000V') 
+		OR metadata->>'make' ILIKE '%xiaoyi%' THEN
+			RETURN 4;
+
+		-- Diff GPS devices
+		ELSIF lower(metadata->>'make') IN ('stfmani', 'trimble', 'imajing')
+		OR metadata->>'model' IN ('LB5') OR metadata->>'model' ILIKE '%ladybug%' THEN
+			RETURN 2;
+
+		-- Smartphones or not-so-good non-diff GPS devices
+		ELSIF lower(metadata->>'make') IN (
+			'samsung', 'xiaomi', 'huawei', 'ricoh', 'lenovo', 'motorola', 'oneplus',
+			'apple', 'google', 'sony', 'wiko', 'asus', 'cubot', 'lge', 'fairphone',
+			'realme', 'symphony', 'crosscall', 'htc', 'homtom', 'hmd global', 'oppo',
+			'ulefone'
+		) THEN
+			RETURN 5;
+		
+		-- Fallback for unknown make/model
+		ELSE
+			RETURN NULL;
+		END IF;
+	-- Fallback : no value
+	ELSE
+		RETURN NULL;
+	END IF;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
+
+
+-- Number of pixels on horizon per FOV degree
+CREATE OR REPLACE FUNCTION h_pixel_density(metadata JSONB) RETURNS INT AS $$
+BEGIN
+	RETURN round((metadata->>'width')::float / (metadata->>'field_of_view')::float);
+END;
+$$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
+
+
+-- Clean-up EXIF, as we're here to rewrite all pictures...
+--   Removes all proprietary keys ending in .0x1234 (hex key)
+CREATE OR REPLACE FUNCTION clean_exif(exif JSONB) RETURNS JSONB AS $$
+	SELECT jsonb_object_agg(key, value)
+	FROM (
+		SELECT key, value
+		FROM jsonb_each(exif)
+		WHERE key !~ '\.0x[0-9a-fA-F]+$'
+	) a;
+$$ LANGUAGE sql IMMUTABLE PARALLEL SAFE;
+
+
+-- Add columns for pictures
+ALTER TABLE pictures
+	ADD COLUMN gps_accuracy_m FLOAT,
+	ADD COLUMN h_pixel_density INT;
+
+
+-- Auto-insert pixel density/GPS accuracy for new pictures
+CREATE OR REPLACE FUNCTION pictures_hpixdens_gpsacc() RETURNS TRIGGER AS $$
+BEGIN
+    NEW.h_pixel_density := h_pixel_density(NEW.metadata);
+	NEW.gps_accuracy_m := gps_accuracy(NEW.metadata, NEW.exif);
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER trg_pictures_hpixdens_gpsacc
+BEFORE INSERT ON pictures
+FOR EACH ROW
+EXECUTE FUNCTION pictures_hpixdens_gpsacc();
+
+CREATE TRIGGER trg_pictures_hpixdens_gpsacc_upd
+BEFORE UPDATE OF metadata, exif ON pictures
+FOR EACH ROW
+EXECUTE FUNCTION pictures_hpixdens_gpsacc();
+
+
+-- Add info on sequences too
+ALTER TABLE sequences
+	ADD COLUMN computed_h_pixel_density INT,
+	ADD COLUMN computed_gps_accuracy FLOAT;

geovisio/migrations/20241017_01_GuOjF-pic-quality-update.rollback.sql

@@ -0,0 +1,4 @@
+-- pic_quality_update
+-- depends: 20241011_01_e1j5C-pic-quality
+
+-- Does nothing

geovisio/migrations/20241017_01_GuOjF-pic-quality-update.sql

@@ -0,0 +1,61 @@
+-- pic_quality_update
+-- depends: 20241011_01_e1j5C-pic-quality
+-- transactional: false
+
+-- Update pictures in batch
+CREATE OR REPLACE PROCEDURE update_pictures_gps_accuracy() AS $$
+DECLARE
+	last_inserted_at TIMESTAMPTZ;
+BEGIN
+	SELECT min(inserted_at) - INTERVAL '1 minute' FROM pictures INTO last_inserted_at;
+
+	WHILE last_inserted_at IS NOT NULL LOOP
+
+		WITH 
+			-- get a batch of 100 000 pictures to update
+			pic_to_update AS (
+				SELECT id, inserted_at from pictures where inserted_at > last_inserted_at ORDER BY inserted_at ASC LIMIT 100000
+			)
+			, updated_pic AS (
+				UPDATE pictures 
+					SET gps_accuracy_m = gps_accuracy(metadata, exif),
+						h_pixel_density = h_pixel_density(missing_fov(metadata)),
+						exif = clean_exif(exif),
+						metadata = missing_fov(metadata)
+					WHERE id in (SELECT id FROM pic_to_update)
+			)
+			SELECT MAX(inserted_at) FROM pic_to_update INTO last_inserted_at;
+		
+		RAISE NOTICE 'max insertion date is now %', last_inserted_at;
+
+		-- commit transaction (as a procedure is in an implicit transaction, it will start a new transaction after this)
+		COMMIT;
+
+	END LOOP;
+	RAISE NOTICE 'update finished';
+END
+$$ LANGUAGE plpgsql;
+
+-- Perform pictures update
+SET session_replication_role = replica;
+CALL update_pictures_gps_accuracy();
+
+-- Update sequences as well
+UPDATE sequences s
+SET
+	computed_h_pixel_density = CASE WHEN array_length(reshpd, 1) = 1 THEN reshpd[1] ELSE NULL END,
+	computed_gps_accuracy = gpsacc
+FROM (
+	SELECT
+		sp.seq_id,
+		ARRAY_AGG(DISTINCT p.h_pixel_density) AS reshpd,
+		PERCENTILE_CONT(0.9) WITHIN GROUP(ORDER BY p.gps_accuracy_m) AS gpsacc
+	FROM sequences_pictures sp
+	JOIN pictures p ON sp.pic_id = p.id
+	GROUP BY sp.seq_id
+) p
+WHERE s.id = p.seq_id;
+
+-- Put back triggers && cleanup
+SET session_replication_role = DEFAULT;
+DROP PROCEDURE update_pictures_gps_accuracy;

geovisio/migrations/20241017_01_RiFlm-pictures-to-delete.rollback.sql

@@ -0,0 +1,43 @@
+-- pictures_to_delete
+-- depends: 20240912_01_dAALm-account-index
+
+ALTER TABLE job_queue DROP COLUMN IF EXISTS picture_to_delete_id;
+-- ALTER TABLE job_queue DROP CONSTRAINT one_external_id;
+ALTER TABLE job_queue ADD CONSTRAINT one_external_id CHECK (num_nonnulls(picture_id, upload_set_id, sequence_id) = 1);
+
+ALTER TABLE job_history DROP COLUMN IF EXISTS picture_to_delete_id;
+-- ALTER TABLE job_history DROP CONSTRAINT one_external_id;
+ALTER TABLE job_history ADD CONSTRAINT one_external_id CHECK (num_nonnulls(picture_id, upload_set_id, sequence_id) = 1);
+
+ALTER TABLE pictures
+	DROP CONSTRAINT upload_set_fk_id ,
+	ADD CONSTRAINT upload_set_fk_id FOREIGN KEY (upload_set_id) REFERENCES upload_sets(id);
+
+ALTER TABLE upload_sets
+ADD COLUMN IF NOT EXISTS deleted BOOLEAN DEFAULT FALSE;
+
+DROP TRIGGER ask_for_all_file_deletion_after_delete_trg ON pictures;
+
+DROP FUNCTION IF EXISTS delete_upload_set_on_last_picture CASCADE;
+CREATE FUNCTION delete_upload_set_on_last_picture() RETURNS trigger AS $$
+BEGIN
+    IF OLD.upload_set_id IS NOT NULL AND NOT EXISTS (
+            SELECT 1 
+            FROM pictures 
+            WHERE upload_set_id = OLD.upload_set_id AND id != OLD.id
+            AND status != 'waiting-for-delete'
+            LIMIT 1
+        ) THEN
+        -- if it's the last picture of an upload set, we delete the upload set
+        UPDATE upload_sets SET deleted = true WHERE id = OLD.upload_set_id AND not deleted;
+        INSERT INTO job_queue AS j (upload_set_id, task)  VALUES (OLD.upload_set_id, 'delete') ON CONFLICT (upload_set_id) DO UPDATE SET task = 'delete' WHERE j.task != 'delete';
+    END IF;
+    RETURN NULL;
+
+END $$ LANGUAGE plpgsql;
+
+CREATE TRIGGER delete_upload_set_on_last_picture_trg
+AFTER DELETE ON pictures
+REFERENCING OLD TABLE AS deleted_pictures
+FOR EACH ROW -- a picture is always deleted by the workers, one by one, so it's useless to run this trigger for each statement
+EXECUTE FUNCTION delete_upload_set_on_last_picture();

geovisio/migrations/20241017_01_RiFlm-pictures-to-delete.sql

@@ -0,0 +1,75 @@
+-- pictures_to_delete
+-- depends: 20240912_01_dAALm-account-index
+
+-- Add another column to the job_queue tabel to store the pictures to delete without using a foreign key, 
+-- so that we can keep them even after their deletion
+ALTER TABLE job_queue DROP CONSTRAINT one_external_id;
+
+ALTER TABLE job_queue ADD COLUMN IF NOT EXISTS picture_to_delete_id UUID;
+
+ALTER TABLE job_queue ADD CONSTRAINT one_external_id CHECK (num_nonnulls(picture_id, upload_set_id, sequence_id, picture_to_delete_id) = 1);
+
+ALTER TABLE job_history ADD COLUMN IF NOT EXISTS picture_to_delete_id UUID;
+ALTER TABLE job_history DROP CONSTRAINT one_external_id;
+ALTER TABLE job_history ADD CONSTRAINT one_external_id CHECK (num_nonnulls(picture_id, upload_set_id, sequence_id, picture_to_delete_id) = 1);
+
+-- when an upload set is deleted, all it's pictures are added to the deletion queue, and deleted from the database (since there is a `ON DELETE CASCADE`)
+
+ALTER TABLE pictures
+	DROP CONSTRAINT upload_set_fk_id ,
+	ADD CONSTRAINT upload_set_fk_id FOREIGN KEY (upload_set_id) REFERENCES upload_sets(id) ON DELETE CASCADE;
+
+-- also update how the last picture trigger the deletion of its upload set
+DROP FUNCTION IF EXISTS delete_upload_set_on_last_picture CASCADE;
+CREATE FUNCTION delete_upload_set_on_last_picture() RETURNS trigger AS $$
+BEGIN
+    IF OLD.upload_set_id IS NOT NULL AND NOT EXISTS (
+            SELECT 1 
+            FROM pictures 
+            WHERE upload_set_id = OLD.upload_set_id AND id != OLD.id
+            AND status != 'waiting-for-delete' -- Note: this status is deprecated, but we need to consider it anyway since it can remain some pictures with this status in the database for the migration
+            LIMIT 1
+        ) THEN
+        -- if it's the last picture of an upload set, we delete the upload set
+        DELETE FROM upload_sets WHERE id = OLD.upload_set_id;
+    END IF;
+    RETURN NULL;
+
+END $$ LANGUAGE plpgsql;
+
+CREATE TRIGGER delete_upload_set_on_last_picture_trg
+AFTER DELETE ON pictures
+REFERENCING OLD TABLE AS deleted_pictures
+FOR EACH ROW -- a picture is always deleted by the workers, one by one, so it's useless to run this trigger for each statement
+EXECUTE FUNCTION delete_upload_set_on_last_picture();
+
+-- Add to the new queue all pictures marked and waiting for delete and all pictures from deleted upload sets
+INSERT INTO job_queue(picture_to_delete_id, task)
+SELECT p.id, 'delete' 
+FROM pictures p 
+LEFT JOIN upload_sets us ON us.id = p.upload_set_id 
+WHERE p.status = 'waiting-for-delete' OR us.deleted;
+
+-- cleanup all pictures/uploadset, as we don't need them anymore since they are added to the new deletion queue
+DELETE FROM pictures WHERE status = 'waiting-for-delete';
+DELETE FROM upload_sets WHERE deleted;
+
+-- the deleted column of upload_sets is now useless, we can delete them right away
+ALTER TABLE upload_sets DROP COLUMN deleted;
+
+DROP FUNCTION IF EXISTS ask_for_all_file_deletion_after_delete;
+CREATE FUNCTION ask_for_all_file_deletion_after_delete() RETURNS trigger AS $$
+BEGIN
+    INSERT INTO job_queue(picture_to_delete_id, task)
+    SELECT p.id, 'delete' FROM deleted_pictures p;
+    RETURN NULL;
+
+END $$ LANGUAGE plpgsql;
+
+CREATE TRIGGER ask_for_all_file_deletion_after_delete_trg
+AFTER DELETE ON pictures
+REFERENCING OLD TABLE AS deleted_pictures
+FOR EACH STATEMENT
+EXECUTE FUNCTION ask_for_all_file_deletion_after_delete();
+
+

geovisio/migrations/20241104_01_yhRVu-rejection-details.rollback.sql

@@ -0,0 +1,4 @@
+-- rejection_details
+-- depends: 20241004_01_d1zfe-pictures-grid-360  20241017_01_RiFlm-pictures-to-delete
+
+ALTER TABLE files DROP COLUMN rejection_details;

geovisio/migrations/20241104_01_yhRVu-rejection-details.sql

@@ -0,0 +1,4 @@
+-- rejection_details
+-- depends: 20241004_01_d1zfe-pictures-grid-360  20241017_01_RiFlm-pictures-to-delete
+
+ALTER TABLE files ADD COLUMN rejection_details JSONB;

geovisio/migrations/20241128_01_ugthx-job-queue-args.rollback.sql

@@ -0,0 +1,25 @@
+-- job_queue_args
+-- depends: 20241017_01_GuOjF-pic-quality-update  20241104_01_yhRVu-rejection-details
+
+ALTER TABLE job_queue DROP COLUMN args;
+ALTER TABLE job_history DROP COLUMN args;
+
+-- put trigger as old value
+DROP TRIGGER trigger_process_picture_insertion ON pictures;
+
+CREATE OR REPLACE FUNCTION picture_insertion() RETURNS TRIGGER AS
+$BODY$
+BEGIN
+    INSERT INTO
+        pictures_to_process(picture_id)
+	VALUES 
+		(new.id);
+	RETURN new;
+END;
+$BODY$
+language plpgsql;
+
+CREATE TRIGGER trigger_process_picture_insertion
+     AFTER INSERT ON pictures
+     FOR EACH ROW
+     EXECUTE PROCEDURE picture_insertion();

geovisio/migrations/20241128_01_ugthx-job-queue-args.sql

@@ -0,0 +1,31 @@
+-- job_queue_args
+-- depends: 20241017_01_GuOjF-pic-quality-update  20241104_01_yhRVu-rejection-details
+
+ALTER TABLE job_queue ADD COLUMN IF NOT EXISTS args JSONB;
+ALTER TABLE job_history ADD COLUMN IF NOT EXISTS args JSONB;
+
+-- Update the trigger that insert each new picture into pictures_to_process
+-- Now we check if the picture has been blurred before upload, to skip the blurring process
+DROP TRIGGER trigger_process_picture_insertion ON pictures;
+CREATE OR REPLACE FUNCTION picture_insertion() RETURNS TRIGGER AS
+$BODY$
+DECLARE
+	args JSONB;
+BEGIN
+    IF new.metadata->>'blurredByAuthor' THEN
+        args := jsonb_build_object('skip_blurring', true);
+    END IF;
+
+    INSERT INTO
+        job_queue(picture_id, task, args)
+	VALUES 
+		(new.id, 'prepare', args);
+	RETURN new;
+END;
+$BODY$
+language plpgsql;
+
+CREATE TRIGGER trigger_process_picture_insertion
+     AFTER INSERT ON pictures
+     FOR EACH ROW
+     EXECUTE PROCEDURE picture_insertion();

geovisio/migrations/20241224_01_xuN6n-delete-upload-set-on-last-picture-trg-statement.rollback.sql

@@ -0,0 +1,27 @@
+-- delete_upload_set_on_last_picture_trg_statement
+-- depends: 20241128_01_ugthx-job-queue-args
+
+--put back old trigger
+
+DROP FUNCTION IF EXISTS delete_upload_set_on_last_picture CASCADE;
+CREATE FUNCTION delete_upload_set_on_last_picture() RETURNS trigger AS $$
+BEGIN
+    IF OLD.upload_set_id IS NOT NULL AND NOT EXISTS (
+            SELECT 1 
+            FROM pictures 
+            WHERE upload_set_id = OLD.upload_set_id AND id != OLD.id
+            AND status != 'waiting-for-delete' 
+            LIMIT 1
+        ) THEN
+        -- if it's the last picture of an upload set, we delete the upload set
+        DELETE FROM upload_sets WHERE id = OLD.upload_set_id;
+    END IF;
+    RETURN NULL;
+
+END $$ LANGUAGE plpgsql;
+
+CREATE TRIGGER delete_upload_set_on_last_picture_trg
+AFTER DELETE ON pictures
+REFERENCING OLD TABLE AS deleted_pictures
+FOR EACH ROW 
+EXECUTE FUNCTION delete_upload_set_on_last_picture();

geovisio/migrations/20241224_01_xuN6n-delete-upload-set-on-last-picture-trg-statement.sql

@@ -0,0 +1,28 @@
+-- delete_upload_set_on_last_picture_trg_statement
+-- depends: 20241128_01_ugthx-job-queue-args
+
+-- Change the trigger to run for each statement, so that we run it once when deleting an uploadset 
+
+DROP FUNCTION IF EXISTS delete_upload_set_on_last_picture CASCADE;
+CREATE FUNCTION delete_upload_set_on_last_picture() RETURNS trigger AS $$
+BEGIN
+    WITH upload_sets AS (
+        SELECT distinct(upload_set_id)
+        FROM deleted_pictures
+        WHERE NOT EXISTS (
+                SELECT 1 
+                FROM pictures p
+                WHERE p.upload_set_id = upload_set_id AND p.id NOT in (SELECT id FROM deleted_pictures)
+                LIMIT 1
+            )
+    )
+    DELETE FROM upload_sets WHERE id IN (SELECT upload_set_id FROM upload_sets);
+    RETURN NULL;
+
+END $$ LANGUAGE plpgsql;
+
+CREATE TRIGGER delete_upload_set_on_last_picture_trg
+AFTER DELETE ON pictures
+REFERENCING OLD TABLE AS deleted_pictures
+FOR EACH STATEMENT
+EXECUTE FUNCTION delete_upload_set_on_last_picture();

geovisio/migrations/20250102_01_EElhA-rm-cameras.rollback.sql

@@ -0,0 +1,126 @@
+-- rm-cameras
+-- depends: 20241128_01_ugthx-job-queue-args
+
+DROP TRIGGER trg_pictures_hpixdens_upd ON pictures;
+DROP TRIGGER trg_pictures_hpixdens ON pictures;
+DROP FUNCTION pictures_hpixdens;
+
+-- Read float values from EXIF
+CREATE OR REPLACE FUNCTION get_float(val VARCHAR) RETURNS FLOAT AS $$
+DECLARE
+	list VARCHAR[];
+BEGIN
+	IF val ~ '^\d+\/\d+$' THEN
+		list := regexp_split_to_array(val, '/');
+		RETURN list[1]::float / list[2]::float;
+    ELSIF val ~ '^\d+(\.\d+)?$' THEN
+        RETURN val::float;
+    ELSE
+        RETURN NULL;
+    END IF;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
+
+
+-- GPS accuracy function
+CREATE OR REPLACE FUNCTION gps_accuracy(metadata JSONB, exif JSONB) RETURNS FLOAT AS $$
+DECLARE
+	gps_dop FLOAT;
+	gps_diff INT;
+	gps_hpos_err FLOAT;
+BEGIN
+	-- Parse GPS DOP, either float or fraction
+	gps_dop := get_float(COALESCE(exif->>'Exif.GPSInfo.GPSDOP', exif->>'Xmp.exif.GPSDOP', ''));
+	gps_diff := (COALESCE(exif->>'Exif.GPSInfo.GPSDifferential', exif->>'Xmp.exif.GPSDifferential'))::INT;
+	gps_hpos_err := get_float(COALESCE(exif->>'Exif.GPSInfo.GPSHPositioningError', exif->>'Xmp.exif.GPSHPositioningError'));
+
+	-- Direct horizontal positioning error in meters -> return as is
+	IF gps_hpos_err IS NOT NULL AND gps_hpos_err > 0 THEN
+		RETURN gps_hpos_err;
+	
+	-- GPS DOP available
+	ELSIF gps_dop IS NOT NULL AND gps_dop > 0 THEN
+		-- With a DGPS -> consider GPS nominal error as 1 meter
+		IF gps_diff = 1 THEN
+			RETURN gps_dop;
+		
+		-- Without DGPS -> consider GPS nominal error as 3 meters in average
+		ELSE
+			RETURN 3 * gps_dop;
+		END IF;
+	
+	-- DGPS -> return 2 meters precision
+	ELSIF gps_diff = 1 THEN
+		RETURN 2;
+
+	-- Approximate guesses based on model
+	ELSIF metadata->>'make' IS NOT NULL OR metadata->>'model' IS NOT NULL THEN
+		-- Good non-diff GPS devices (best case is 3m, so setting 4 for tolerance)
+		IF lower(metadata->>'make') IN (
+			'gopro', 'insta360', 'garmin', 'viofo', 'xiaoyi', 'blackvue', 'tectectec',
+			'arashi vision'
+		)
+		OR metadata->>'model' IN ('LG-R105', 'FDR-X1000V') 
+		OR metadata->>'make' ILIKE '%xiaoyi%' THEN
+			RETURN 4;
+
+		-- Diff GPS devices
+		ELSIF lower(metadata->>'make') IN ('stfmani', 'trimble', 'imajing')
+		OR metadata->>'model' IN ('LB5') OR metadata->>'model' ILIKE '%ladybug%' THEN
+			RETURN 2;
+
+		-- Smartphones or not-so-good non-diff GPS devices
+		ELSIF lower(metadata->>'make') IN (
+			'samsung', 'xiaomi', 'huawei', 'ricoh', 'lenovo', 'motorola', 'oneplus',
+			'apple', 'google', 'sony', 'wiko', 'asus', 'cubot', 'lge', 'fairphone',
+			'realme', 'symphony', 'crosscall', 'htc', 'homtom', 'hmd global', 'oppo',
+			'ulefone'
+		) THEN
+			RETURN 5;
+		
+		-- Fallback for unknown make/model
+		ELSE
+			RETURN NULL;
+		END IF;
+	-- Fallback : no value
+	ELSE
+		RETURN NULL;
+	END IF;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
+
+-- Function should not be used in rollback, created a mock one for db_migration tests
+CREATE OR REPLACE FUNCTION missing_fov(metadata JSONB) RETURNS JSONB AS $$
+DECLARE
+	sensor_width FLOAT;
+BEGIN
+	RETURN metadata;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
+
+-- Auto-insert pixel density/GPS accuracy for new pictures
+CREATE OR REPLACE FUNCTION pictures_hpixdens_gpsacc() RETURNS TRIGGER AS $$
+BEGIN
+    NEW.h_pixel_density := h_pixel_density(NEW.metadata);
+	NEW.gps_accuracy_m := gps_accuracy(NEW.metadata, NEW.exif);
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER trg_pictures_hpixdens_gpsacc
+BEFORE INSERT ON pictures
+FOR EACH ROW
+EXECUTE FUNCTION pictures_hpixdens_gpsacc();
+
+CREATE TRIGGER trg_pictures_hpixdens_gpsacc_upd
+BEFORE UPDATE OF metadata, exif ON pictures
+FOR EACH ROW
+EXECUTE FUNCTION pictures_hpixdens_gpsacc();
+
+-- Mock table for db_migration tests
+CREATE TABLE cameras(
+	model VARCHAR PRIMARY KEY,
+	sensor_width FLOAT NOT NULL
+);
+
+CREATE INDEX cameras_model_idx ON cameras USING GIST(model gist_trgm_ops);