geovisio 2.6.0__py3-none-any.whl → 2.7.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. geovisio/__init__.py +36 -7
  2. geovisio/admin_cli/cleanup.py +2 -2
  3. geovisio/admin_cli/db.py +1 -4
  4. geovisio/config_app.py +40 -1
  5. geovisio/db_migrations.py +24 -3
  6. geovisio/templates/main.html +13 -13
  7. geovisio/templates/viewer.html +3 -3
  8. geovisio/translations/de/LC_MESSAGES/messages.mo +0 -0
  9. geovisio/translations/de/LC_MESSAGES/messages.po +804 -0
  10. geovisio/translations/el/LC_MESSAGES/messages.mo +0 -0
  11. geovisio/translations/el/LC_MESSAGES/messages.po +685 -0
  12. geovisio/translations/en/LC_MESSAGES/messages.mo +0 -0
  13. geovisio/translations/en/LC_MESSAGES/messages.po +738 -0
  14. geovisio/translations/es/LC_MESSAGES/messages.mo +0 -0
  15. geovisio/translations/es/LC_MESSAGES/messages.po +778 -0
  16. geovisio/translations/fi/LC_MESSAGES/messages.mo +0 -0
  17. geovisio/translations/fi/LC_MESSAGES/messages.po +589 -0
  18. geovisio/translations/fr/LC_MESSAGES/messages.mo +0 -0
  19. geovisio/translations/fr/LC_MESSAGES/messages.po +814 -0
  20. geovisio/translations/hu/LC_MESSAGES/messages.mo +0 -0
  21. geovisio/translations/hu/LC_MESSAGES/messages.po +773 -0
  22. geovisio/translations/ko/LC_MESSAGES/messages.mo +0 -0
  23. geovisio/translations/ko/LC_MESSAGES/messages.po +685 -0
  24. geovisio/translations/messages.pot +694 -0
  25. geovisio/translations/nl/LC_MESSAGES/messages.mo +0 -0
  26. geovisio/translations/nl/LC_MESSAGES/messages.po +602 -0
  27. geovisio/utils/__init__.py +1 -1
  28. geovisio/utils/auth.py +50 -11
  29. geovisio/utils/db.py +65 -0
  30. geovisio/utils/excluded_areas.py +83 -0
  31. geovisio/utils/extent.py +30 -0
  32. geovisio/utils/fields.py +1 -1
  33. geovisio/utils/filesystems.py +0 -1
  34. geovisio/utils/link.py +14 -0
  35. geovisio/utils/params.py +20 -0
  36. geovisio/utils/pictures.py +110 -88
  37. geovisio/utils/reports.py +171 -0
  38. geovisio/utils/sequences.py +262 -126
  39. geovisio/utils/tokens.py +37 -42
  40. geovisio/utils/upload_set.py +642 -0
  41. geovisio/web/auth.py +37 -37
  42. geovisio/web/collections.py +304 -304
  43. geovisio/web/configuration.py +14 -0
  44. geovisio/web/docs.py +276 -15
  45. geovisio/web/excluded_areas.py +377 -0
  46. geovisio/web/items.py +169 -112
  47. geovisio/web/map.py +104 -36
  48. geovisio/web/params.py +69 -26
  49. geovisio/web/pictures.py +14 -31
  50. geovisio/web/reports.py +399 -0
  51. geovisio/web/rss.py +13 -7
  52. geovisio/web/stac.py +129 -134
  53. geovisio/web/tokens.py +98 -109
  54. geovisio/web/upload_set.py +771 -0
  55. geovisio/web/users.py +100 -73
  56. geovisio/web/utils.py +28 -9
  57. geovisio/workers/runner_pictures.py +241 -207
  58. {geovisio-2.6.0.dist-info → geovisio-2.7.1.dist-info}/METADATA +17 -14
  59. geovisio-2.7.1.dist-info/RECORD +70 -0
  60. {geovisio-2.6.0.dist-info → geovisio-2.7.1.dist-info}/WHEEL +1 -1
  61. geovisio-2.6.0.dist-info/RECORD +0 -41
  62. {geovisio-2.6.0.dist-info → geovisio-2.7.1.dist-info}/LICENSE +0 -0
@@ -1,14 +1,16 @@
1
1
  import psycopg
2
2
  from flask import current_app, url_for
3
+ from flask_babel import gettext as _
3
4
  from psycopg.types.json import Jsonb
4
- from psycopg import sql
5
- from psycopg.sql import SQL
5
+ from psycopg.sql import SQL, Composable
6
6
  from psycopg.rows import dict_row
7
7
  from dataclasses import dataclass, field
8
8
  from typing import Any, List, Dict, Optional
9
9
  import datetime
10
10
  from uuid import UUID
11
11
  from enum import Enum
12
+ from geovisio.utils import db
13
+ from geovisio.utils.auth import Account
12
14
  from geovisio.utils.fields import FieldMapping, SortBy, SQLDirection, BBox, Bounds
13
15
  from geopic_tag_reader import reader
14
16
  from pathlib import PurePath
@@ -17,30 +19,26 @@ import logging
17
19
  import sentry_sdk
18
20
 
19
21
 
20
- def createSequence(metadata, accountId) -> str:
21
- with psycopg.connect(current_app.config["DB_URL"]) as conn:
22
- with conn.cursor() as cursor:
23
- # Add sequence in database
24
- seqId = cursor.execute(
25
- "INSERT INTO sequences(account_id, metadata) VALUES(%s, %s) RETURNING id", [accountId, Jsonb(metadata)]
26
- ).fetchone()
27
-
28
- # Make changes definitive in database
29
- conn.commit()
30
-
31
- if seqId is None:
32
- raise Exception(f"impossible to insert sequence in database")
33
- return seqId[0]
22
+ def createSequence(metadata, accountId, user_agent: Optional[str] = None) -> UUID:
23
+ with db.execute(
24
+ current_app,
25
+ "INSERT INTO sequences(account_id, metadata, user_agent) VALUES(%s, %s, %s) RETURNING id",
26
+ [accountId, Jsonb(metadata), user_agent],
27
+ ) as r:
28
+ seqId = r.fetchone()
29
+ if seqId is None:
30
+ raise Exception("impossible to insert sequence in database")
31
+ return seqId[0]
34
32
 
35
33
 
36
34
  # Mappings from stac name to SQL names
37
35
  STAC_FIELD_MAPPINGS = {
38
36
  p.stac: p
39
37
  for p in [
40
- FieldMapping(sql_column=sql.SQL("inserted_at"), stac="created"),
41
- FieldMapping(sql_column=sql.SQL("updated_at"), stac="updated"),
42
- FieldMapping(sql_column=sql.SQL("computed_capture_date"), stac="datetime"),
43
- FieldMapping(sql_column=sql.SQL("status"), stac="status"),
38
+ FieldMapping(sql_column=SQL("inserted_at"), stac="created"),
39
+ FieldMapping(sql_column=SQL("updated_at"), stac="updated"),
40
+ FieldMapping(sql_column=SQL("computed_capture_date"), stac="datetime"),
41
+ FieldMapping(sql_column=SQL("status"), stac="status"),
44
42
  ]
45
43
  }
46
44
  STAC_FIELD_TO_SQL_FILTER = {p.stac: p.sql_filter.as_string(None) for p in STAC_FIELD_MAPPINGS.values()}
@@ -66,8 +64,8 @@ class CollectionsRequest:
66
64
  created_before: Optional[datetime.datetime] = None
67
65
  user_id: Optional[UUID] = None
68
66
  bbox: Optional[BBox] = None
69
- user_filter: Optional[sql.SQL] = None
70
- pagination_filter: Optional[sql.SQL] = None
67
+ user_filter: Optional[SQL] = None
68
+ pagination_filter: Optional[SQL] = None
71
69
  limit: int = 100
72
70
  userOwnsAllCollections: bool = False # bool to represent that the user's asking for the collections is the owner of them
73
71
 
@@ -77,9 +75,8 @@ class CollectionsRequest:
77
75
 
78
76
  def get_collections(request: CollectionsRequest) -> Collections:
79
77
  # Check basic parameters
80
- seq_filter: List[sql.Composable] = []
78
+ seq_filter: List[Composable] = []
81
79
  seq_params: dict = {}
82
- pic_filter = [SQL("sp.seq_id = s.id")]
83
80
 
84
81
  # Sort-by parameter
85
82
  # Note for review: I'm not sure I understand this non nullity constraint, but if so, shouldn't all sortby fields be added ?
@@ -92,25 +89,25 @@ def get_collections(request: CollectionsRequest) -> Collections:
92
89
  seq_filter.append(SQL("s.account_id = %(account)s"))
93
90
  seq_params["account"] = request.user_id
94
91
 
95
- if request.user_filter is None or "status" not in request.user_filter.as_string(None):
92
+ user_filter_str = request.user_filter.as_string(None) if request.user_filter is not None else None
93
+ if user_filter_str is None or "status" not in user_filter_str:
96
94
  # if the filter does not contains any `status` condition, we want to show only 'ready' collection to the general users, and non deleted one for the owner
97
95
  if not request.userOwnsAllCollections:
98
- seq_filter.append(SQL("s.status = 'ready'"))
99
- pic_filter.append(SQL("p.status = 'ready'"))
96
+ seq_filter.append(SQL("status = 'ready'"))
100
97
  else:
101
- seq_filter.append(SQL("s.status != 'deleted'"))
98
+ seq_filter.append(SQL("status != 'deleted'"))
102
99
  else:
103
- # else, even if there are status filter, we make sure not to show hidden pictures/sequence to non owner
104
- if not request.userOwnsAllCollections:
105
- seq_filter.append(SQL("s.status <> 'hidden'"))
106
- pic_filter.append(SQL("p.status <> 'hidden'"))
100
+ if not request.userOwnsAllCollections and "'deleted'" not in user_filter_str:
101
+ # if there are status filter and we ask for deleted sequence, we also include hidden one and consider them as deleted
102
+ seq_filter.append(SQL("status <> 'hidden'"))
107
103
 
108
104
  status_field = None
109
105
  if request.userOwnsAllCollections:
110
106
  # only logged users can see detailed status
111
107
  status_field = SQL("s.status AS status")
112
108
  else:
113
- status_field = SQL("CASE WHEN s.status = 'deleted' THEN s.status ELSE NULL END AS status")
109
+ # hidden sequence are marked as deleted, this way crawler can update their catalog
110
+ status_field = SQL("CASE WHEN s.status IN ('hidden', 'deleted') THEN 'deleted' ELSE s.status END AS status")
114
111
 
115
112
  # Datetime
116
113
  if request.min_dt is not None:
@@ -136,94 +133,92 @@ def get_collections(request: CollectionsRequest) -> Collections:
136
133
  seq_filter.append(SQL("s.inserted_at < %(created_before)s::timestamp with time zone"))
137
134
  seq_params["created_before"] = request.created_before
138
135
 
139
- with psycopg.connect(current_app.config["DB_URL"], row_factory=dict_row) as conn:
140
- with conn.cursor() as cursor:
141
- sqlSequencesRaw = SQL(
142
- """
143
- SELECT
144
- s.id,
145
- s.status,
146
- s.metadata->>'title' AS name,
147
- s.inserted_at AS created,
148
- s.updated_at AS updated,
149
- ST_XMin(s.bbox) AS minx,
150
- ST_YMin(s.bbox) AS miny,
151
- ST_XMax(s.bbox) AS maxx,
152
- ST_YMax(s.bbox) AS maxy,
153
- accounts.name AS account_name,
154
- ST_X(ST_PointN(s.geom, 1)) AS x1,
155
- ST_Y(ST_PointN(s.geom, 1)) AS y1,
156
- s.min_picture_ts AS mints,
157
- s.max_picture_ts AS maxts,
158
- s.nb_pictures AS nbpic,
159
- {status},
160
- s.computed_capture_date AS datetime
161
- FROM sequences s
162
- LEFT JOIN accounts on s.account_id = accounts.id
163
- WHERE {filter}
164
- ORDER BY {order1}
165
- LIMIT {limit}
136
+ with utils.db.cursor(current_app, row_factory=dict_row) as cursor:
137
+ sqlSequencesRaw = SQL(
166
138
  """
139
+ SELECT
140
+ s.id,
141
+ s.status,
142
+ s.metadata->>'title' AS name,
143
+ s.inserted_at AS created,
144
+ s.updated_at AS updated,
145
+ ST_XMin(s.bbox) AS minx,
146
+ ST_YMin(s.bbox) AS miny,
147
+ ST_XMax(s.bbox) AS maxx,
148
+ ST_YMax(s.bbox) AS maxy,
149
+ accounts.name AS account_name,
150
+ s.account_id AS account_id,
151
+ ST_X(ST_PointN(ST_GeometryN(s.geom, 1), 1)) AS x1,
152
+ ST_Y(ST_PointN(ST_GeometryN(s.geom, 1), 1)) AS y1,
153
+ s.min_picture_ts AS mints,
154
+ s.max_picture_ts AS maxts,
155
+ s.nb_pictures AS nbpic,
156
+ {status},
157
+ s.computed_capture_date AS datetime,
158
+ s.user_agent,
159
+ ROUND(ST_Length(s.geom::geography)) / 1000 AS length_km,
160
+ s.computed_h_pixel_density,
161
+ s.computed_gps_accuracy
162
+ FROM sequences s
163
+ LEFT JOIN accounts on s.account_id = accounts.id
164
+ WHERE {filter}
165
+ ORDER BY {order1}
166
+ LIMIT {limit}
167
+ """
168
+ )
169
+ sqlSequences = sqlSequencesRaw.format(
170
+ filter=SQL(" AND ").join(seq_filter),
171
+ order1=request.sort_by.as_sql(),
172
+ limit=request.limit,
173
+ status=status_field,
174
+ )
175
+
176
+ # Different request if we want the last n sequences
177
+ # Useful for paginating from last page to first
178
+ if request.pagination_filter and (
179
+ (
180
+ request.sort_by.fields[0].direction == SQLDirection.ASC
181
+ and request.pagination_filter.as_string(None).startswith(f"({request.sort_by.fields[0].field.sql_filter.as_string(None)} <")
182
+ )
183
+ or (
184
+ request.sort_by.fields[0].direction == SQLDirection.DESC
185
+ and request.pagination_filter.as_string(None).startswith(f"({request.sort_by.fields[0].field.sql_filter.as_string(None)} >")
167
186
  )
168
- sqlSequences = sqlSequencesRaw.format(
187
+ ):
188
+ base_query = sqlSequencesRaw.format(
169
189
  filter=SQL(" AND ").join(seq_filter),
170
- order1=request.sort_by.as_sql(),
190
+ order1=request.sort_by.revert(),
171
191
  limit=request.limit,
172
- pic_filter=SQL(" AND ").join(pic_filter),
173
192
  status=status_field,
174
193
  )
175
-
176
- # Different request if we want the last n sequences
177
- # Useful for paginating from last page to first
178
- if request.pagination_filter and (
179
- (
180
- request.sort_by.fields[0].direction == SQLDirection.ASC
181
- and request.pagination_filter.as_string(None).startswith(
182
- f"({request.sort_by.fields[0].field.sql_filter.as_string(None)} <"
183
- )
184
- )
185
- or (
186
- request.sort_by.fields[0].direction == SQLDirection.DESC
187
- and request.pagination_filter.as_string(None).startswith(
188
- f"({request.sort_by.fields[0].field.sql_filter.as_string(None)} >"
189
- )
190
- )
191
- ):
192
- base_query = sqlSequencesRaw.format(
193
- filter=SQL(" AND ").join(seq_filter),
194
- order1=request.sort_by.revert(),
195
- limit=request.limit,
196
- pic_filter=SQL(" AND ").join(pic_filter),
197
- status=status_field,
198
- )
199
- sqlSequences = SQL(
200
- """
201
- SELECT *
202
- FROM ({base_query}) s
203
- ORDER BY {order2}
194
+ sqlSequences = SQL(
204
195
  """
205
- ).format(
206
- order2=request.sort_by.as_sql(),
207
- base_query=base_query,
208
- )
209
-
210
- records = cursor.execute(sqlSequences, seq_params).fetchall()
211
-
212
- query_bounds = None
213
- for s in records:
214
- first_order_val = s.get(request.sort_by.fields[0].field.stac)
215
- if first_order_val is None:
216
- continue
217
- if query_bounds is None:
218
- query_bounds = Bounds(first_order_val, first_order_val)
219
- else:
220
- query_bounds.update(first_order_val)
221
-
222
- return Collections(
223
- collections=records,
224
- query_first_order_bounds=query_bounds,
196
+ SELECT *
197
+ FROM ({base_query}) s
198
+ ORDER BY {order2}
199
+ """
200
+ ).format(
201
+ order2=request.sort_by.as_sql(),
202
+ base_query=base_query,
225
203
  )
226
204
 
205
+ records = cursor.execute(sqlSequences, seq_params).fetchall()
206
+
207
+ query_bounds = None
208
+ for s in records:
209
+ first_order_val = s.get(request.sort_by.fields[0].field.stac)
210
+ if first_order_val is None:
211
+ continue
212
+ if query_bounds is None:
213
+ query_bounds = Bounds(first_order_val, first_order_val)
214
+ else:
215
+ query_bounds.update(first_order_val)
216
+
217
+ return Collections(
218
+ collections=records,
219
+ query_first_order_bounds=query_bounds,
220
+ )
221
+
227
222
 
228
223
  def get_pagination_links(
229
224
  route: str,
@@ -381,17 +376,23 @@ def sort_collection(db, collectionId: UUID, sortby: CollectionSort):
381
376
 
382
377
  if usedDateField is None:
383
378
  raise errors.InvalidAPIUsage(
384
- "Sort by file date is not possible on this sequence (no file date information available on pictures)",
379
+ _("Sort by file date is not possible on this sequence (no file date information available on pictures)"),
385
380
  status_code=422,
386
381
  )
387
382
 
388
383
  for pm in picMetas:
389
384
  # Find value for wanted sort
390
385
  if sortby.order == CollectionSortOrder.GPS_DATE:
391
- pm["sort"] = reader.decodeGPSDateTime(pm["exif"], "Exif.GPSInfo")[0]
386
+ if "ts_gps" in pm["metadata"]:
387
+ pm["sort"] = pm["metadata"]["ts_gps"]
388
+ else:
389
+ pm["sort"] = reader.decodeGPSDateTime(pm["exif"], "Exif.GPSInfo", _)[0]
392
390
  elif sortby.order == CollectionSortOrder.FILE_DATE:
393
- assert usedDateField # nullity has been checked before
394
- pm["sort"] = reader.decodeDateTimeOriginal(pm["exif"], usedDateField)[0]
391
+ if "ts_camera" in pm["metadata"]:
392
+ pm["sort"] = pm["metadata"]["ts_camera"]
393
+ else:
394
+ assert usedDateField # nullity has been checked before
395
+ pm["sort"] = reader.decodeDateTimeOriginal(pm["exif"], usedDateField, _)[0]
395
396
  elif sortby.order == CollectionSortOrder.FILE_NAME:
396
397
  pm["sort"] = pm["metadata"].get("originalFileName")
397
398
  if isFileNameNumeric:
@@ -400,7 +401,11 @@ def sort_collection(db, collectionId: UUID, sortby: CollectionSort):
400
401
  # Fail if sort value is missing
401
402
  if pm["sort"] is None:
402
403
  raise errors.InvalidAPIUsage(
403
- f"Sort using {sortby} is not possible on this sequence, picture {pm['id']} is missing mandatory metadata",
404
+ _(
405
+ "Sort using %(sort)s is not possible on this sequence, picture %(pic)s is missing mandatory metadata",
406
+ sort=sortby,
407
+ pic=pm["id"],
408
+ ),
404
409
  status_code=422,
405
410
  )
406
411
 
@@ -476,10 +481,72 @@ def update_headings(
476
481
  )
477
482
 
478
483
 
479
- def update_pictures_grid(db) -> bool:
480
- """Refreshes the pictures_grid materialized view for an up-to-date view of pictures availability on map.
484
+ def add_finalization_job(cursor, seqId: UUID):
485
+ """
486
+ Add a sequence finalization job in the queue.
487
+ If there is already a finalization job, do nothing (changing it might cause a deadlock, since a worker could be processing this job)
488
+ """
489
+ cursor.execute(
490
+ """INSERT INTO
491
+ job_queue(sequence_id, task)
492
+ VALUES (%(seq_id)s, 'finalize')
493
+ ON CONFLICT (sequence_id) DO NOTHING""",
494
+ {"seq_id": seqId},
495
+ )
481
496
 
482
- Note: the transaction is not commited at the end, you need to commit it or use an autocommit connection.
497
+
498
+ def finalize(cursor, seqId: UUID, logger: logging.Logger = logging.getLogger()):
499
+ """
500
+ Finalize a sequence, by updating its status and computed fields.
501
+ """
502
+ with sentry_sdk.start_span(description="Finalizing sequence") as span:
503
+ span.set_data("sequence_id", seqId)
504
+ logger.debug(f"Finalizing sequence {seqId}")
505
+
506
+ with utils.time.log_elapsed(f"Finalizing sequence {seqId}"):
507
+ # Complete missing headings in pictures
508
+ update_headings(cursor, seqId)
509
+
510
+ # Change sequence database status in DB
511
+ # Also generates data in computed columns
512
+ cursor.execute(
513
+ """WITH
514
+ aggregated_pictures AS (
515
+ SELECT
516
+ sp.seq_id,
517
+ MIN(p.ts::DATE) AS day,
518
+ ARRAY_AGG(DISTINCT TRIM(
519
+ CONCAT(p.metadata->>'make', ' ', p.metadata->>'model')
520
+ )) AS models,
521
+ ARRAY_AGG(DISTINCT p.metadata->>'type') AS types,
522
+ ARRAY_AGG(DISTINCT p.h_pixel_density) AS reshpd,
523
+ PERCENTILE_CONT(0.9) WITHIN GROUP(ORDER BY p.gps_accuracy_m) AS gpsacc
524
+ FROM sequences_pictures sp
525
+ JOIN pictures p ON sp.pic_id = p.id
526
+ WHERE sp.seq_id = %(seq)s
527
+ GROUP BY sp.seq_id
528
+ )
529
+ UPDATE sequences
530
+ SET
531
+ status = CASE WHEN status = 'hidden' THEN 'hidden'::sequence_status ELSE 'ready'::sequence_status END, -- we don't want to change status if it's hidden
532
+ geom = compute_sequence_geom(id),
533
+ bbox = compute_sequence_bbox(id),
534
+ computed_type = CASE WHEN array_length(types, 1) = 1 THEN types[1] ELSE NULL END,
535
+ computed_model = CASE WHEN array_length(models, 1) = 1 THEN models[1] ELSE NULL END,
536
+ computed_capture_date = day,
537
+ computed_h_pixel_density = CASE WHEN array_length(reshpd, 1) = 1 THEN reshpd[1] ELSE NULL END,
538
+ computed_gps_accuracy = gpsacc
539
+ FROM aggregated_pictures
540
+ WHERE id = %(seq)s
541
+ """,
542
+ {"seq": seqId},
543
+ )
544
+
545
+ logger.info(f"Sequence {seqId} is ready")
546
+
547
+
548
+ def update_pictures_grid() -> Optional[datetime.datetime]:
549
+ """Refreshes the pictures_grid materialized view for an up-to-date view of pictures availability on map.
483
550
 
484
551
  Parameters
485
552
  ----------
@@ -490,17 +557,86 @@ def update_pictures_grid(db) -> bool:
490
557
  -------
491
558
  bool : True if the view has been updated else False
492
559
  """
560
+ from geovisio.utils import db
561
+
493
562
  logger = logging.getLogger("geovisio.picture_grid")
494
- with db.transaction():
563
+
564
+ # get a connection outside of the connection pool in order to avoid
565
+ # the default statement timeout as this query can be very long
566
+ with db.long_queries_conn(current_app) as conn, conn.transaction():
495
567
  try:
496
- db.execute("SELECT refreshed_at FROM refresh_database FOR UPDATE NOWAIT").fetchone()
568
+ conn.execute("SELECT refreshed_at FROM refresh_database FOR UPDATE NOWAIT").fetchone()
497
569
  except psycopg.errors.LockNotAvailable:
498
570
  logger.info("Database refresh already in progress, nothing to do")
499
571
  return False
500
572
 
501
- with sentry_sdk.start_span(description="Refreshing database") as span:
502
- with utils.time.log_elapsed(f"Refreshing database", logger=logger):
573
+ with sentry_sdk.start_span(description="Refreshing database"):
574
+ with utils.time.log_elapsed("Refreshing database", logger=logger):
503
575
  logger.info("Refreshing database")
504
- db.execute("UPDATE refresh_database SET refreshed_at = NOW()")
505
- db.execute("REFRESH MATERIALIZED VIEW pictures_grid")
576
+ conn.execute("UPDATE refresh_database SET refreshed_at = NOW()")
577
+ conn.execute("REFRESH MATERIALIZED VIEW CONCURRENTLY pictures_grid")
578
+
506
579
  return True
580
+
581
+
582
+ def delete_collection(collectionId: UUID, account: Optional[Account]) -> int:
583
+ """
584
+ Mark a collection as deleted and delete all it's pictures.
585
+
586
+ Note that since the deletion as asynchronous, some workers need to be run in order for the deletion to be effective.
587
+ """
588
+ with db.conn(current_app) as conn:
589
+ with conn.transaction(), conn.cursor() as cursor:
590
+ sequence = cursor.execute(
591
+ "SELECT status, account_id FROM sequences WHERE id = %s AND status != 'deleted'", [collectionId]
592
+ ).fetchone()
593
+
594
+ # sequence not found
595
+ if not sequence:
596
+ raise errors.InvalidAPIUsage(_("Collection %(c)s wasn't found in database", c=collectionId), status_code=404)
597
+
598
+ # Account associated to sequence doesn't match current user
599
+ if account is not None and account.id != str(sequence[1]):
600
+ raise errors.InvalidAPIUsage("You're not authorized to edit this sequence", status_code=403)
601
+
602
+ logging.info(f"Asking for deletion of sequence {collectionId} and all its pictures")
603
+
604
+ # mark all the pictures as waiting for deletion for async removal as this can be quite long if the storage is slow and there are lots of pictures
605
+ # Note: To avoid a deadlock if some workers are currently also working on those picture to prepare them,
606
+ # the SQL queries are split in 2:
607
+ # - First a query to remove jobs preparing those pictures
608
+ # - Then a query deleting those pictures from the database (and a trigger will add async deletion tasks to the queue)
609
+ #
610
+ # Since the workers lock their job_queue row when working, at the end of this query, we know that there are no more workers working on those pictures,
611
+ # so we can delete them without fearing a deadlock.
612
+ cursor.execute(
613
+ """WITH pic2rm AS (
614
+ SELECT pic_id FROM sequences_pictures WHERE seq_id = %(seq)s
615
+ ),
616
+ picWithoutOtherSeq AS (
617
+ SELECT pic_id FROM pic2rm
618
+ EXCEPT
619
+ SELECT pic_id FROM sequences_pictures WHERE pic_id IN (SELECT pic_id FROM pic2rm) AND seq_id != %(seq)s
620
+ )
621
+ DELETE FROM job_queue WHERE picture_id IN (SELECT pic_id FROM picWithoutOtherSeq)""",
622
+ {"seq": collectionId},
623
+ ).rowcount
624
+ # if there was a finalize task for this collection in the queue, we remove it, it's useless
625
+ cursor.execute("""DELETE FROM job_queue WHERE sequence_id = %(seq)s""", {"seq": collectionId})
626
+
627
+ # after the task have been added to the queue, delete the pictures, and db triggers will ensure the correct deletion jobs are added
628
+ nb_updated = cursor.execute(
629
+ """WITH pic2rm AS (
630
+ SELECT pic_id FROM sequences_pictures WHERE seq_id = %(seq)s
631
+ ),
632
+ picWithoutOtherSeq AS (
633
+ SELECT pic_id FROM pic2rm
634
+ EXCEPT
635
+ SELECT pic_id FROM sequences_pictures WHERE pic_id IN (SELECT pic_id FROM pic2rm) AND seq_id != %(seq)s
636
+ )
637
+ DELETE FROM pictures WHERE id IN (SELECT pic_id FROM picWithoutOtherSeq)""",
638
+ {"seq": collectionId},
639
+ ).rowcount
640
+
641
+ cursor.execute("UPDATE sequences SET status = 'deleted' WHERE id = %s", [collectionId])
642
+ return nb_updated
geovisio/utils/tokens.py CHANGED
@@ -1,11 +1,9 @@
1
1
  from geovisio import errors
2
- from geovisio.utils import auth
2
+ from geovisio.utils import auth, db
3
3
  from geovisio.web.tokens import _decode_jwt_token, _generate_jwt_token
4
-
5
-
6
- import psycopg
7
4
  from authlib.jose.errors import BadSignatureError
8
5
  from flask import current_app
6
+ from flask_babel import gettext as _
9
7
  from psycopg.rows import dict_row
10
8
 
11
9
 
@@ -14,7 +12,7 @@ import logging
14
12
 
15
13
  class InvalidTokenException(errors.InvalidAPIUsage):
16
14
  def __init__(self, details, status_code=401):
17
- msg = f"Token not valid"
15
+ msg = "Token not valid"
18
16
  super().__init__(msg, status_code=status_code, payload={"details": {"error": details}})
19
17
 
20
18
 
@@ -39,39 +37,37 @@ def get_account_from_jwt_token(jwt_token: str) -> auth.Account:
39
37
  """
40
38
  try:
41
39
  decoded = _decode_jwt_token(jwt_token)
42
- except BadSignatureError as e:
40
+ except BadSignatureError:
43
41
  logging.exception("invalid signature of jwt token")
44
- raise InvalidTokenException("JWT token signature does not match")
42
+ raise InvalidTokenException(_("JWT token signature does not match"))
45
43
  token_id = decoded["sub"]
46
44
 
47
- with psycopg.connect(current_app.config["DB_URL"], row_factory=dict_row) as conn:
48
- with conn.cursor() as cursor:
49
- # check token existence
50
- records = cursor.execute(
51
- """
52
- SELECT
53
- t.account_id AS id, a.name, a.oauth_provider, a.oauth_id
54
- FROM tokens t
55
- LEFT OUTER JOIN accounts a ON t.account_id = a.id
56
- WHERE t.id = %(token)s
57
- """,
58
- {"token": token_id},
59
- ).fetchone()
60
- if not records:
61
- raise InvalidTokenException("Token does not exist anymore", status_code=403)
62
-
63
- if not records["id"]:
64
- raise InvalidTokenException(
65
- "Token not yet claimed, this token cannot be used yet. Either claim this token or generate a new one", status_code=403
66
- )
67
-
68
- return auth.Account(
69
- id=str(records["id"]),
70
- name=records["name"],
71
- oauth_provider=records["oauth_provider"],
72
- oauth_id=records["oauth_id"],
45
+ with db.cursor(current_app, row_factory=dict_row) as cursor:
46
+ # check token existence
47
+ records = cursor.execute(
48
+ """SELECT
49
+ t.account_id AS id, a.name, a.oauth_provider, a.oauth_id, a.role
50
+ FROM tokens t
51
+ LEFT OUTER JOIN accounts a ON t.account_id = a.id
52
+ WHERE t.id = %(token)s""",
53
+ {"token": token_id},
54
+ ).fetchone()
55
+ if not records:
56
+ raise InvalidTokenException(_("Token does not exist anymore"), status_code=403)
57
+
58
+ if not records["id"]:
59
+ raise InvalidTokenException(
60
+ _("Token not yet claimed, this token cannot be used yet. Either claim this token or generate a new one"), status_code=403
73
61
  )
74
62
 
63
+ return auth.Account(
64
+ id=str(records["id"]),
65
+ name=records["name"],
66
+ oauth_provider=records["oauth_provider"],
67
+ oauth_id=records["oauth_id"],
68
+ role=auth.AccountRole[records["role"]],
69
+ )
70
+
75
71
 
76
72
  def get_default_account_jwt_token() -> str:
77
73
  """
@@ -80,18 +76,17 @@ def get_default_account_jwt_token() -> str:
80
76
  Note: do not expose this method externally, only an instance administrator should be able to get the default account JWT token!
81
77
  """
82
78
 
83
- with psycopg.connect(current_app.config["DB_URL"], row_factory=dict_row) as conn:
84
- with conn.cursor() as cursor:
85
- # check token existence
86
- records = cursor.execute(
87
- """
79
+ with db.cursor(current_app, row_factory=dict_row) as cursor:
80
+ # check token existence
81
+ records = cursor.execute(
82
+ """
88
83
  SELECT t.id AS id
89
84
  FROM tokens t
90
85
  JOIN accounts a ON t.account_id = a.id
91
86
  WHERE a.is_default
92
87
  """
93
- ).fetchone()
94
- if not records:
95
- raise Exception("Default account has no associated token")
88
+ ).fetchone()
89
+ if not records:
90
+ raise Exception("Default account has no associated token")
96
91
 
97
- return _generate_jwt_token(records["id"])
92
+ return _generate_jwt_token(records["id"])