geovisio 2.6.0__py3-none-any.whl → 2.7.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- geovisio/__init__.py +36 -7
- geovisio/admin_cli/cleanup.py +2 -2
- geovisio/admin_cli/db.py +1 -4
- geovisio/config_app.py +40 -1
- geovisio/db_migrations.py +24 -3
- geovisio/templates/main.html +13 -13
- geovisio/templates/viewer.html +3 -3
- geovisio/translations/de/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/de/LC_MESSAGES/messages.po +804 -0
- geovisio/translations/el/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/el/LC_MESSAGES/messages.po +685 -0
- geovisio/translations/en/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/en/LC_MESSAGES/messages.po +738 -0
- geovisio/translations/es/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/es/LC_MESSAGES/messages.po +778 -0
- geovisio/translations/fi/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/fi/LC_MESSAGES/messages.po +589 -0
- geovisio/translations/fr/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/fr/LC_MESSAGES/messages.po +814 -0
- geovisio/translations/hu/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/hu/LC_MESSAGES/messages.po +773 -0
- geovisio/translations/ko/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/ko/LC_MESSAGES/messages.po +685 -0
- geovisio/translations/messages.pot +694 -0
- geovisio/translations/nl/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/nl/LC_MESSAGES/messages.po +602 -0
- geovisio/utils/__init__.py +1 -1
- geovisio/utils/auth.py +50 -11
- geovisio/utils/db.py +65 -0
- geovisio/utils/excluded_areas.py +83 -0
- geovisio/utils/extent.py +30 -0
- geovisio/utils/fields.py +1 -1
- geovisio/utils/filesystems.py +0 -1
- geovisio/utils/link.py +14 -0
- geovisio/utils/params.py +20 -0
- geovisio/utils/pictures.py +110 -88
- geovisio/utils/reports.py +171 -0
- geovisio/utils/sequences.py +262 -126
- geovisio/utils/tokens.py +37 -42
- geovisio/utils/upload_set.py +642 -0
- geovisio/web/auth.py +37 -37
- geovisio/web/collections.py +304 -304
- geovisio/web/configuration.py +14 -0
- geovisio/web/docs.py +276 -15
- geovisio/web/excluded_areas.py +377 -0
- geovisio/web/items.py +169 -112
- geovisio/web/map.py +104 -36
- geovisio/web/params.py +69 -26
- geovisio/web/pictures.py +14 -31
- geovisio/web/reports.py +399 -0
- geovisio/web/rss.py +13 -7
- geovisio/web/stac.py +129 -134
- geovisio/web/tokens.py +98 -109
- geovisio/web/upload_set.py +771 -0
- geovisio/web/users.py +100 -73
- geovisio/web/utils.py +28 -9
- geovisio/workers/runner_pictures.py +241 -207
- {geovisio-2.6.0.dist-info → geovisio-2.7.1.dist-info}/METADATA +17 -14
- geovisio-2.7.1.dist-info/RECORD +70 -0
- {geovisio-2.6.0.dist-info → geovisio-2.7.1.dist-info}/WHEEL +1 -1
- geovisio-2.6.0.dist-info/RECORD +0 -41
- {geovisio-2.6.0.dist-info → geovisio-2.7.1.dist-info}/LICENSE +0 -0
geovisio/utils/sequences.py
CHANGED
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
import psycopg
|
|
2
2
|
from flask import current_app, url_for
|
|
3
|
+
from flask_babel import gettext as _
|
|
3
4
|
from psycopg.types.json import Jsonb
|
|
4
|
-
from psycopg import
|
|
5
|
-
from psycopg.sql import SQL
|
|
5
|
+
from psycopg.sql import SQL, Composable
|
|
6
6
|
from psycopg.rows import dict_row
|
|
7
7
|
from dataclasses import dataclass, field
|
|
8
8
|
from typing import Any, List, Dict, Optional
|
|
9
9
|
import datetime
|
|
10
10
|
from uuid import UUID
|
|
11
11
|
from enum import Enum
|
|
12
|
+
from geovisio.utils import db
|
|
13
|
+
from geovisio.utils.auth import Account
|
|
12
14
|
from geovisio.utils.fields import FieldMapping, SortBy, SQLDirection, BBox, Bounds
|
|
13
15
|
from geopic_tag_reader import reader
|
|
14
16
|
from pathlib import PurePath
|
|
@@ -17,30 +19,26 @@ import logging
|
|
|
17
19
|
import sentry_sdk
|
|
18
20
|
|
|
19
21
|
|
|
20
|
-
def createSequence(metadata, accountId) ->
|
|
21
|
-
with
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
if seqId is None:
|
|
32
|
-
raise Exception(f"impossible to insert sequence in database")
|
|
33
|
-
return seqId[0]
|
|
22
|
+
def createSequence(metadata, accountId, user_agent: Optional[str] = None) -> UUID:
|
|
23
|
+
with db.execute(
|
|
24
|
+
current_app,
|
|
25
|
+
"INSERT INTO sequences(account_id, metadata, user_agent) VALUES(%s, %s, %s) RETURNING id",
|
|
26
|
+
[accountId, Jsonb(metadata), user_agent],
|
|
27
|
+
) as r:
|
|
28
|
+
seqId = r.fetchone()
|
|
29
|
+
if seqId is None:
|
|
30
|
+
raise Exception("impossible to insert sequence in database")
|
|
31
|
+
return seqId[0]
|
|
34
32
|
|
|
35
33
|
|
|
36
34
|
# Mappings from stac name to SQL names
|
|
37
35
|
STAC_FIELD_MAPPINGS = {
|
|
38
36
|
p.stac: p
|
|
39
37
|
for p in [
|
|
40
|
-
FieldMapping(sql_column=
|
|
41
|
-
FieldMapping(sql_column=
|
|
42
|
-
FieldMapping(sql_column=
|
|
43
|
-
FieldMapping(sql_column=
|
|
38
|
+
FieldMapping(sql_column=SQL("inserted_at"), stac="created"),
|
|
39
|
+
FieldMapping(sql_column=SQL("updated_at"), stac="updated"),
|
|
40
|
+
FieldMapping(sql_column=SQL("computed_capture_date"), stac="datetime"),
|
|
41
|
+
FieldMapping(sql_column=SQL("status"), stac="status"),
|
|
44
42
|
]
|
|
45
43
|
}
|
|
46
44
|
STAC_FIELD_TO_SQL_FILTER = {p.stac: p.sql_filter.as_string(None) for p in STAC_FIELD_MAPPINGS.values()}
|
|
@@ -66,8 +64,8 @@ class CollectionsRequest:
|
|
|
66
64
|
created_before: Optional[datetime.datetime] = None
|
|
67
65
|
user_id: Optional[UUID] = None
|
|
68
66
|
bbox: Optional[BBox] = None
|
|
69
|
-
user_filter: Optional[
|
|
70
|
-
pagination_filter: Optional[
|
|
67
|
+
user_filter: Optional[SQL] = None
|
|
68
|
+
pagination_filter: Optional[SQL] = None
|
|
71
69
|
limit: int = 100
|
|
72
70
|
userOwnsAllCollections: bool = False # bool to represent that the user's asking for the collections is the owner of them
|
|
73
71
|
|
|
@@ -77,9 +75,8 @@ class CollectionsRequest:
|
|
|
77
75
|
|
|
78
76
|
def get_collections(request: CollectionsRequest) -> Collections:
|
|
79
77
|
# Check basic parameters
|
|
80
|
-
seq_filter: List[
|
|
78
|
+
seq_filter: List[Composable] = []
|
|
81
79
|
seq_params: dict = {}
|
|
82
|
-
pic_filter = [SQL("sp.seq_id = s.id")]
|
|
83
80
|
|
|
84
81
|
# Sort-by parameter
|
|
85
82
|
# Note for review: I'm not sure I understand this non nullity constraint, but if so, shouldn't all sortby fields be added ?
|
|
@@ -92,25 +89,25 @@ def get_collections(request: CollectionsRequest) -> Collections:
|
|
|
92
89
|
seq_filter.append(SQL("s.account_id = %(account)s"))
|
|
93
90
|
seq_params["account"] = request.user_id
|
|
94
91
|
|
|
95
|
-
if request.user_filter is None
|
|
92
|
+
user_filter_str = request.user_filter.as_string(None) if request.user_filter is not None else None
|
|
93
|
+
if user_filter_str is None or "status" not in user_filter_str:
|
|
96
94
|
# if the filter does not contains any `status` condition, we want to show only 'ready' collection to the general users, and non deleted one for the owner
|
|
97
95
|
if not request.userOwnsAllCollections:
|
|
98
|
-
seq_filter.append(SQL("
|
|
99
|
-
pic_filter.append(SQL("p.status = 'ready'"))
|
|
96
|
+
seq_filter.append(SQL("status = 'ready'"))
|
|
100
97
|
else:
|
|
101
|
-
seq_filter.append(SQL("
|
|
98
|
+
seq_filter.append(SQL("status != 'deleted'"))
|
|
102
99
|
else:
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
seq_filter.append(SQL("
|
|
106
|
-
pic_filter.append(SQL("p.status <> 'hidden'"))
|
|
100
|
+
if not request.userOwnsAllCollections and "'deleted'" not in user_filter_str:
|
|
101
|
+
# if there are status filter and we ask for deleted sequence, we also include hidden one and consider them as deleted
|
|
102
|
+
seq_filter.append(SQL("status <> 'hidden'"))
|
|
107
103
|
|
|
108
104
|
status_field = None
|
|
109
105
|
if request.userOwnsAllCollections:
|
|
110
106
|
# only logged users can see detailed status
|
|
111
107
|
status_field = SQL("s.status AS status")
|
|
112
108
|
else:
|
|
113
|
-
|
|
109
|
+
# hidden sequence are marked as deleted, this way crawler can update their catalog
|
|
110
|
+
status_field = SQL("CASE WHEN s.status IN ('hidden', 'deleted') THEN 'deleted' ELSE s.status END AS status")
|
|
114
111
|
|
|
115
112
|
# Datetime
|
|
116
113
|
if request.min_dt is not None:
|
|
@@ -136,94 +133,92 @@ def get_collections(request: CollectionsRequest) -> Collections:
|
|
|
136
133
|
seq_filter.append(SQL("s.inserted_at < %(created_before)s::timestamp with time zone"))
|
|
137
134
|
seq_params["created_before"] = request.created_before
|
|
138
135
|
|
|
139
|
-
with
|
|
140
|
-
|
|
141
|
-
sqlSequencesRaw = SQL(
|
|
142
|
-
"""
|
|
143
|
-
SELECT
|
|
144
|
-
s.id,
|
|
145
|
-
s.status,
|
|
146
|
-
s.metadata->>'title' AS name,
|
|
147
|
-
s.inserted_at AS created,
|
|
148
|
-
s.updated_at AS updated,
|
|
149
|
-
ST_XMin(s.bbox) AS minx,
|
|
150
|
-
ST_YMin(s.bbox) AS miny,
|
|
151
|
-
ST_XMax(s.bbox) AS maxx,
|
|
152
|
-
ST_YMax(s.bbox) AS maxy,
|
|
153
|
-
accounts.name AS account_name,
|
|
154
|
-
ST_X(ST_PointN(s.geom, 1)) AS x1,
|
|
155
|
-
ST_Y(ST_PointN(s.geom, 1)) AS y1,
|
|
156
|
-
s.min_picture_ts AS mints,
|
|
157
|
-
s.max_picture_ts AS maxts,
|
|
158
|
-
s.nb_pictures AS nbpic,
|
|
159
|
-
{status},
|
|
160
|
-
s.computed_capture_date AS datetime
|
|
161
|
-
FROM sequences s
|
|
162
|
-
LEFT JOIN accounts on s.account_id = accounts.id
|
|
163
|
-
WHERE {filter}
|
|
164
|
-
ORDER BY {order1}
|
|
165
|
-
LIMIT {limit}
|
|
136
|
+
with utils.db.cursor(current_app, row_factory=dict_row) as cursor:
|
|
137
|
+
sqlSequencesRaw = SQL(
|
|
166
138
|
"""
|
|
139
|
+
SELECT
|
|
140
|
+
s.id,
|
|
141
|
+
s.status,
|
|
142
|
+
s.metadata->>'title' AS name,
|
|
143
|
+
s.inserted_at AS created,
|
|
144
|
+
s.updated_at AS updated,
|
|
145
|
+
ST_XMin(s.bbox) AS minx,
|
|
146
|
+
ST_YMin(s.bbox) AS miny,
|
|
147
|
+
ST_XMax(s.bbox) AS maxx,
|
|
148
|
+
ST_YMax(s.bbox) AS maxy,
|
|
149
|
+
accounts.name AS account_name,
|
|
150
|
+
s.account_id AS account_id,
|
|
151
|
+
ST_X(ST_PointN(ST_GeometryN(s.geom, 1), 1)) AS x1,
|
|
152
|
+
ST_Y(ST_PointN(ST_GeometryN(s.geom, 1), 1)) AS y1,
|
|
153
|
+
s.min_picture_ts AS mints,
|
|
154
|
+
s.max_picture_ts AS maxts,
|
|
155
|
+
s.nb_pictures AS nbpic,
|
|
156
|
+
{status},
|
|
157
|
+
s.computed_capture_date AS datetime,
|
|
158
|
+
s.user_agent,
|
|
159
|
+
ROUND(ST_Length(s.geom::geography)) / 1000 AS length_km,
|
|
160
|
+
s.computed_h_pixel_density,
|
|
161
|
+
s.computed_gps_accuracy
|
|
162
|
+
FROM sequences s
|
|
163
|
+
LEFT JOIN accounts on s.account_id = accounts.id
|
|
164
|
+
WHERE {filter}
|
|
165
|
+
ORDER BY {order1}
|
|
166
|
+
LIMIT {limit}
|
|
167
|
+
"""
|
|
168
|
+
)
|
|
169
|
+
sqlSequences = sqlSequencesRaw.format(
|
|
170
|
+
filter=SQL(" AND ").join(seq_filter),
|
|
171
|
+
order1=request.sort_by.as_sql(),
|
|
172
|
+
limit=request.limit,
|
|
173
|
+
status=status_field,
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
# Different request if we want the last n sequences
|
|
177
|
+
# Useful for paginating from last page to first
|
|
178
|
+
if request.pagination_filter and (
|
|
179
|
+
(
|
|
180
|
+
request.sort_by.fields[0].direction == SQLDirection.ASC
|
|
181
|
+
and request.pagination_filter.as_string(None).startswith(f"({request.sort_by.fields[0].field.sql_filter.as_string(None)} <")
|
|
182
|
+
)
|
|
183
|
+
or (
|
|
184
|
+
request.sort_by.fields[0].direction == SQLDirection.DESC
|
|
185
|
+
and request.pagination_filter.as_string(None).startswith(f"({request.sort_by.fields[0].field.sql_filter.as_string(None)} >")
|
|
167
186
|
)
|
|
168
|
-
|
|
187
|
+
):
|
|
188
|
+
base_query = sqlSequencesRaw.format(
|
|
169
189
|
filter=SQL(" AND ").join(seq_filter),
|
|
170
|
-
order1=request.sort_by.
|
|
190
|
+
order1=request.sort_by.revert(),
|
|
171
191
|
limit=request.limit,
|
|
172
|
-
pic_filter=SQL(" AND ").join(pic_filter),
|
|
173
192
|
status=status_field,
|
|
174
193
|
)
|
|
175
|
-
|
|
176
|
-
# Different request if we want the last n sequences
|
|
177
|
-
# Useful for paginating from last page to first
|
|
178
|
-
if request.pagination_filter and (
|
|
179
|
-
(
|
|
180
|
-
request.sort_by.fields[0].direction == SQLDirection.ASC
|
|
181
|
-
and request.pagination_filter.as_string(None).startswith(
|
|
182
|
-
f"({request.sort_by.fields[0].field.sql_filter.as_string(None)} <"
|
|
183
|
-
)
|
|
184
|
-
)
|
|
185
|
-
or (
|
|
186
|
-
request.sort_by.fields[0].direction == SQLDirection.DESC
|
|
187
|
-
and request.pagination_filter.as_string(None).startswith(
|
|
188
|
-
f"({request.sort_by.fields[0].field.sql_filter.as_string(None)} >"
|
|
189
|
-
)
|
|
190
|
-
)
|
|
191
|
-
):
|
|
192
|
-
base_query = sqlSequencesRaw.format(
|
|
193
|
-
filter=SQL(" AND ").join(seq_filter),
|
|
194
|
-
order1=request.sort_by.revert(),
|
|
195
|
-
limit=request.limit,
|
|
196
|
-
pic_filter=SQL(" AND ").join(pic_filter),
|
|
197
|
-
status=status_field,
|
|
198
|
-
)
|
|
199
|
-
sqlSequences = SQL(
|
|
200
|
-
"""
|
|
201
|
-
SELECT *
|
|
202
|
-
FROM ({base_query}) s
|
|
203
|
-
ORDER BY {order2}
|
|
194
|
+
sqlSequences = SQL(
|
|
204
195
|
"""
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
query_bounds = None
|
|
213
|
-
for s in records:
|
|
214
|
-
first_order_val = s.get(request.sort_by.fields[0].field.stac)
|
|
215
|
-
if first_order_val is None:
|
|
216
|
-
continue
|
|
217
|
-
if query_bounds is None:
|
|
218
|
-
query_bounds = Bounds(first_order_val, first_order_val)
|
|
219
|
-
else:
|
|
220
|
-
query_bounds.update(first_order_val)
|
|
221
|
-
|
|
222
|
-
return Collections(
|
|
223
|
-
collections=records,
|
|
224
|
-
query_first_order_bounds=query_bounds,
|
|
196
|
+
SELECT *
|
|
197
|
+
FROM ({base_query}) s
|
|
198
|
+
ORDER BY {order2}
|
|
199
|
+
"""
|
|
200
|
+
).format(
|
|
201
|
+
order2=request.sort_by.as_sql(),
|
|
202
|
+
base_query=base_query,
|
|
225
203
|
)
|
|
226
204
|
|
|
205
|
+
records = cursor.execute(sqlSequences, seq_params).fetchall()
|
|
206
|
+
|
|
207
|
+
query_bounds = None
|
|
208
|
+
for s in records:
|
|
209
|
+
first_order_val = s.get(request.sort_by.fields[0].field.stac)
|
|
210
|
+
if first_order_val is None:
|
|
211
|
+
continue
|
|
212
|
+
if query_bounds is None:
|
|
213
|
+
query_bounds = Bounds(first_order_val, first_order_val)
|
|
214
|
+
else:
|
|
215
|
+
query_bounds.update(first_order_val)
|
|
216
|
+
|
|
217
|
+
return Collections(
|
|
218
|
+
collections=records,
|
|
219
|
+
query_first_order_bounds=query_bounds,
|
|
220
|
+
)
|
|
221
|
+
|
|
227
222
|
|
|
228
223
|
def get_pagination_links(
|
|
229
224
|
route: str,
|
|
@@ -381,17 +376,23 @@ def sort_collection(db, collectionId: UUID, sortby: CollectionSort):
|
|
|
381
376
|
|
|
382
377
|
if usedDateField is None:
|
|
383
378
|
raise errors.InvalidAPIUsage(
|
|
384
|
-
"Sort by file date is not possible on this sequence (no file date information available on pictures)",
|
|
379
|
+
_("Sort by file date is not possible on this sequence (no file date information available on pictures)"),
|
|
385
380
|
status_code=422,
|
|
386
381
|
)
|
|
387
382
|
|
|
388
383
|
for pm in picMetas:
|
|
389
384
|
# Find value for wanted sort
|
|
390
385
|
if sortby.order == CollectionSortOrder.GPS_DATE:
|
|
391
|
-
|
|
386
|
+
if "ts_gps" in pm["metadata"]:
|
|
387
|
+
pm["sort"] = pm["metadata"]["ts_gps"]
|
|
388
|
+
else:
|
|
389
|
+
pm["sort"] = reader.decodeGPSDateTime(pm["exif"], "Exif.GPSInfo", _)[0]
|
|
392
390
|
elif sortby.order == CollectionSortOrder.FILE_DATE:
|
|
393
|
-
|
|
394
|
-
|
|
391
|
+
if "ts_camera" in pm["metadata"]:
|
|
392
|
+
pm["sort"] = pm["metadata"]["ts_camera"]
|
|
393
|
+
else:
|
|
394
|
+
assert usedDateField # nullity has been checked before
|
|
395
|
+
pm["sort"] = reader.decodeDateTimeOriginal(pm["exif"], usedDateField, _)[0]
|
|
395
396
|
elif sortby.order == CollectionSortOrder.FILE_NAME:
|
|
396
397
|
pm["sort"] = pm["metadata"].get("originalFileName")
|
|
397
398
|
if isFileNameNumeric:
|
|
@@ -400,7 +401,11 @@ def sort_collection(db, collectionId: UUID, sortby: CollectionSort):
|
|
|
400
401
|
# Fail if sort value is missing
|
|
401
402
|
if pm["sort"] is None:
|
|
402
403
|
raise errors.InvalidAPIUsage(
|
|
403
|
-
|
|
404
|
+
_(
|
|
405
|
+
"Sort using %(sort)s is not possible on this sequence, picture %(pic)s is missing mandatory metadata",
|
|
406
|
+
sort=sortby,
|
|
407
|
+
pic=pm["id"],
|
|
408
|
+
),
|
|
404
409
|
status_code=422,
|
|
405
410
|
)
|
|
406
411
|
|
|
@@ -476,10 +481,72 @@ def update_headings(
|
|
|
476
481
|
)
|
|
477
482
|
|
|
478
483
|
|
|
479
|
-
def
|
|
480
|
-
"""
|
|
484
|
+
def add_finalization_job(cursor, seqId: UUID):
|
|
485
|
+
"""
|
|
486
|
+
Add a sequence finalization job in the queue.
|
|
487
|
+
If there is already a finalization job, do nothing (changing it might cause a deadlock, since a worker could be processing this job)
|
|
488
|
+
"""
|
|
489
|
+
cursor.execute(
|
|
490
|
+
"""INSERT INTO
|
|
491
|
+
job_queue(sequence_id, task)
|
|
492
|
+
VALUES (%(seq_id)s, 'finalize')
|
|
493
|
+
ON CONFLICT (sequence_id) DO NOTHING""",
|
|
494
|
+
{"seq_id": seqId},
|
|
495
|
+
)
|
|
481
496
|
|
|
482
|
-
|
|
497
|
+
|
|
498
|
+
def finalize(cursor, seqId: UUID, logger: logging.Logger = logging.getLogger()):
|
|
499
|
+
"""
|
|
500
|
+
Finalize a sequence, by updating its status and computed fields.
|
|
501
|
+
"""
|
|
502
|
+
with sentry_sdk.start_span(description="Finalizing sequence") as span:
|
|
503
|
+
span.set_data("sequence_id", seqId)
|
|
504
|
+
logger.debug(f"Finalizing sequence {seqId}")
|
|
505
|
+
|
|
506
|
+
with utils.time.log_elapsed(f"Finalizing sequence {seqId}"):
|
|
507
|
+
# Complete missing headings in pictures
|
|
508
|
+
update_headings(cursor, seqId)
|
|
509
|
+
|
|
510
|
+
# Change sequence database status in DB
|
|
511
|
+
# Also generates data in computed columns
|
|
512
|
+
cursor.execute(
|
|
513
|
+
"""WITH
|
|
514
|
+
aggregated_pictures AS (
|
|
515
|
+
SELECT
|
|
516
|
+
sp.seq_id,
|
|
517
|
+
MIN(p.ts::DATE) AS day,
|
|
518
|
+
ARRAY_AGG(DISTINCT TRIM(
|
|
519
|
+
CONCAT(p.metadata->>'make', ' ', p.metadata->>'model')
|
|
520
|
+
)) AS models,
|
|
521
|
+
ARRAY_AGG(DISTINCT p.metadata->>'type') AS types,
|
|
522
|
+
ARRAY_AGG(DISTINCT p.h_pixel_density) AS reshpd,
|
|
523
|
+
PERCENTILE_CONT(0.9) WITHIN GROUP(ORDER BY p.gps_accuracy_m) AS gpsacc
|
|
524
|
+
FROM sequences_pictures sp
|
|
525
|
+
JOIN pictures p ON sp.pic_id = p.id
|
|
526
|
+
WHERE sp.seq_id = %(seq)s
|
|
527
|
+
GROUP BY sp.seq_id
|
|
528
|
+
)
|
|
529
|
+
UPDATE sequences
|
|
530
|
+
SET
|
|
531
|
+
status = CASE WHEN status = 'hidden' THEN 'hidden'::sequence_status ELSE 'ready'::sequence_status END, -- we don't want to change status if it's hidden
|
|
532
|
+
geom = compute_sequence_geom(id),
|
|
533
|
+
bbox = compute_sequence_bbox(id),
|
|
534
|
+
computed_type = CASE WHEN array_length(types, 1) = 1 THEN types[1] ELSE NULL END,
|
|
535
|
+
computed_model = CASE WHEN array_length(models, 1) = 1 THEN models[1] ELSE NULL END,
|
|
536
|
+
computed_capture_date = day,
|
|
537
|
+
computed_h_pixel_density = CASE WHEN array_length(reshpd, 1) = 1 THEN reshpd[1] ELSE NULL END,
|
|
538
|
+
computed_gps_accuracy = gpsacc
|
|
539
|
+
FROM aggregated_pictures
|
|
540
|
+
WHERE id = %(seq)s
|
|
541
|
+
""",
|
|
542
|
+
{"seq": seqId},
|
|
543
|
+
)
|
|
544
|
+
|
|
545
|
+
logger.info(f"Sequence {seqId} is ready")
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def update_pictures_grid() -> Optional[datetime.datetime]:
|
|
549
|
+
"""Refreshes the pictures_grid materialized view for an up-to-date view of pictures availability on map.
|
|
483
550
|
|
|
484
551
|
Parameters
|
|
485
552
|
----------
|
|
@@ -490,17 +557,86 @@ def update_pictures_grid(db) -> bool:
|
|
|
490
557
|
-------
|
|
491
558
|
bool : True if the view has been updated else False
|
|
492
559
|
"""
|
|
560
|
+
from geovisio.utils import db
|
|
561
|
+
|
|
493
562
|
logger = logging.getLogger("geovisio.picture_grid")
|
|
494
|
-
|
|
563
|
+
|
|
564
|
+
# get a connection outside of the connection pool in order to avoid
|
|
565
|
+
# the default statement timeout as this query can be very long
|
|
566
|
+
with db.long_queries_conn(current_app) as conn, conn.transaction():
|
|
495
567
|
try:
|
|
496
|
-
|
|
568
|
+
conn.execute("SELECT refreshed_at FROM refresh_database FOR UPDATE NOWAIT").fetchone()
|
|
497
569
|
except psycopg.errors.LockNotAvailable:
|
|
498
570
|
logger.info("Database refresh already in progress, nothing to do")
|
|
499
571
|
return False
|
|
500
572
|
|
|
501
|
-
with sentry_sdk.start_span(description="Refreshing database")
|
|
502
|
-
with utils.time.log_elapsed(
|
|
573
|
+
with sentry_sdk.start_span(description="Refreshing database"):
|
|
574
|
+
with utils.time.log_elapsed("Refreshing database", logger=logger):
|
|
503
575
|
logger.info("Refreshing database")
|
|
504
|
-
|
|
505
|
-
|
|
576
|
+
conn.execute("UPDATE refresh_database SET refreshed_at = NOW()")
|
|
577
|
+
conn.execute("REFRESH MATERIALIZED VIEW CONCURRENTLY pictures_grid")
|
|
578
|
+
|
|
506
579
|
return True
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def delete_collection(collectionId: UUID, account: Optional[Account]) -> int:
|
|
583
|
+
"""
|
|
584
|
+
Mark a collection as deleted and delete all it's pictures.
|
|
585
|
+
|
|
586
|
+
Note that since the deletion as asynchronous, some workers need to be run in order for the deletion to be effective.
|
|
587
|
+
"""
|
|
588
|
+
with db.conn(current_app) as conn:
|
|
589
|
+
with conn.transaction(), conn.cursor() as cursor:
|
|
590
|
+
sequence = cursor.execute(
|
|
591
|
+
"SELECT status, account_id FROM sequences WHERE id = %s AND status != 'deleted'", [collectionId]
|
|
592
|
+
).fetchone()
|
|
593
|
+
|
|
594
|
+
# sequence not found
|
|
595
|
+
if not sequence:
|
|
596
|
+
raise errors.InvalidAPIUsage(_("Collection %(c)s wasn't found in database", c=collectionId), status_code=404)
|
|
597
|
+
|
|
598
|
+
# Account associated to sequence doesn't match current user
|
|
599
|
+
if account is not None and account.id != str(sequence[1]):
|
|
600
|
+
raise errors.InvalidAPIUsage("You're not authorized to edit this sequence", status_code=403)
|
|
601
|
+
|
|
602
|
+
logging.info(f"Asking for deletion of sequence {collectionId} and all its pictures")
|
|
603
|
+
|
|
604
|
+
# mark all the pictures as waiting for deletion for async removal as this can be quite long if the storage is slow and there are lots of pictures
|
|
605
|
+
# Note: To avoid a deadlock if some workers are currently also working on those picture to prepare them,
|
|
606
|
+
# the SQL queries are split in 2:
|
|
607
|
+
# - First a query to remove jobs preparing those pictures
|
|
608
|
+
# - Then a query deleting those pictures from the database (and a trigger will add async deletion tasks to the queue)
|
|
609
|
+
#
|
|
610
|
+
# Since the workers lock their job_queue row when working, at the end of this query, we know that there are no more workers working on those pictures,
|
|
611
|
+
# so we can delete them without fearing a deadlock.
|
|
612
|
+
cursor.execute(
|
|
613
|
+
"""WITH pic2rm AS (
|
|
614
|
+
SELECT pic_id FROM sequences_pictures WHERE seq_id = %(seq)s
|
|
615
|
+
),
|
|
616
|
+
picWithoutOtherSeq AS (
|
|
617
|
+
SELECT pic_id FROM pic2rm
|
|
618
|
+
EXCEPT
|
|
619
|
+
SELECT pic_id FROM sequences_pictures WHERE pic_id IN (SELECT pic_id FROM pic2rm) AND seq_id != %(seq)s
|
|
620
|
+
)
|
|
621
|
+
DELETE FROM job_queue WHERE picture_id IN (SELECT pic_id FROM picWithoutOtherSeq)""",
|
|
622
|
+
{"seq": collectionId},
|
|
623
|
+
).rowcount
|
|
624
|
+
# if there was a finalize task for this collection in the queue, we remove it, it's useless
|
|
625
|
+
cursor.execute("""DELETE FROM job_queue WHERE sequence_id = %(seq)s""", {"seq": collectionId})
|
|
626
|
+
|
|
627
|
+
# after the task have been added to the queue, delete the pictures, and db triggers will ensure the correct deletion jobs are added
|
|
628
|
+
nb_updated = cursor.execute(
|
|
629
|
+
"""WITH pic2rm AS (
|
|
630
|
+
SELECT pic_id FROM sequences_pictures WHERE seq_id = %(seq)s
|
|
631
|
+
),
|
|
632
|
+
picWithoutOtherSeq AS (
|
|
633
|
+
SELECT pic_id FROM pic2rm
|
|
634
|
+
EXCEPT
|
|
635
|
+
SELECT pic_id FROM sequences_pictures WHERE pic_id IN (SELECT pic_id FROM pic2rm) AND seq_id != %(seq)s
|
|
636
|
+
)
|
|
637
|
+
DELETE FROM pictures WHERE id IN (SELECT pic_id FROM picWithoutOtherSeq)""",
|
|
638
|
+
{"seq": collectionId},
|
|
639
|
+
).rowcount
|
|
640
|
+
|
|
641
|
+
cursor.execute("UPDATE sequences SET status = 'deleted' WHERE id = %s", [collectionId])
|
|
642
|
+
return nb_updated
|
geovisio/utils/tokens.py
CHANGED
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
from geovisio import errors
|
|
2
|
-
from geovisio.utils import auth
|
|
2
|
+
from geovisio.utils import auth, db
|
|
3
3
|
from geovisio.web.tokens import _decode_jwt_token, _generate_jwt_token
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
import psycopg
|
|
7
4
|
from authlib.jose.errors import BadSignatureError
|
|
8
5
|
from flask import current_app
|
|
6
|
+
from flask_babel import gettext as _
|
|
9
7
|
from psycopg.rows import dict_row
|
|
10
8
|
|
|
11
9
|
|
|
@@ -14,7 +12,7 @@ import logging
|
|
|
14
12
|
|
|
15
13
|
class InvalidTokenException(errors.InvalidAPIUsage):
|
|
16
14
|
def __init__(self, details, status_code=401):
|
|
17
|
-
msg =
|
|
15
|
+
msg = "Token not valid"
|
|
18
16
|
super().__init__(msg, status_code=status_code, payload={"details": {"error": details}})
|
|
19
17
|
|
|
20
18
|
|
|
@@ -39,39 +37,37 @@ def get_account_from_jwt_token(jwt_token: str) -> auth.Account:
|
|
|
39
37
|
"""
|
|
40
38
|
try:
|
|
41
39
|
decoded = _decode_jwt_token(jwt_token)
|
|
42
|
-
except BadSignatureError
|
|
40
|
+
except BadSignatureError:
|
|
43
41
|
logging.exception("invalid signature of jwt token")
|
|
44
|
-
raise InvalidTokenException("JWT token signature does not match")
|
|
42
|
+
raise InvalidTokenException(_("JWT token signature does not match"))
|
|
45
43
|
token_id = decoded["sub"]
|
|
46
44
|
|
|
47
|
-
with
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
if not records["id"]:
|
|
64
|
-
raise InvalidTokenException(
|
|
65
|
-
"Token not yet claimed, this token cannot be used yet. Either claim this token or generate a new one", status_code=403
|
|
66
|
-
)
|
|
67
|
-
|
|
68
|
-
return auth.Account(
|
|
69
|
-
id=str(records["id"]),
|
|
70
|
-
name=records["name"],
|
|
71
|
-
oauth_provider=records["oauth_provider"],
|
|
72
|
-
oauth_id=records["oauth_id"],
|
|
45
|
+
with db.cursor(current_app, row_factory=dict_row) as cursor:
|
|
46
|
+
# check token existence
|
|
47
|
+
records = cursor.execute(
|
|
48
|
+
"""SELECT
|
|
49
|
+
t.account_id AS id, a.name, a.oauth_provider, a.oauth_id, a.role
|
|
50
|
+
FROM tokens t
|
|
51
|
+
LEFT OUTER JOIN accounts a ON t.account_id = a.id
|
|
52
|
+
WHERE t.id = %(token)s""",
|
|
53
|
+
{"token": token_id},
|
|
54
|
+
).fetchone()
|
|
55
|
+
if not records:
|
|
56
|
+
raise InvalidTokenException(_("Token does not exist anymore"), status_code=403)
|
|
57
|
+
|
|
58
|
+
if not records["id"]:
|
|
59
|
+
raise InvalidTokenException(
|
|
60
|
+
_("Token not yet claimed, this token cannot be used yet. Either claim this token or generate a new one"), status_code=403
|
|
73
61
|
)
|
|
74
62
|
|
|
63
|
+
return auth.Account(
|
|
64
|
+
id=str(records["id"]),
|
|
65
|
+
name=records["name"],
|
|
66
|
+
oauth_provider=records["oauth_provider"],
|
|
67
|
+
oauth_id=records["oauth_id"],
|
|
68
|
+
role=auth.AccountRole[records["role"]],
|
|
69
|
+
)
|
|
70
|
+
|
|
75
71
|
|
|
76
72
|
def get_default_account_jwt_token() -> str:
|
|
77
73
|
"""
|
|
@@ -80,18 +76,17 @@ def get_default_account_jwt_token() -> str:
|
|
|
80
76
|
Note: do not expose this method externally, only an instance administrator should be able to get the default account JWT token!
|
|
81
77
|
"""
|
|
82
78
|
|
|
83
|
-
with
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
"""
|
|
79
|
+
with db.cursor(current_app, row_factory=dict_row) as cursor:
|
|
80
|
+
# check token existence
|
|
81
|
+
records = cursor.execute(
|
|
82
|
+
"""
|
|
88
83
|
SELECT t.id AS id
|
|
89
84
|
FROM tokens t
|
|
90
85
|
JOIN accounts a ON t.account_id = a.id
|
|
91
86
|
WHERE a.is_default
|
|
92
87
|
"""
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
88
|
+
).fetchone()
|
|
89
|
+
if not records:
|
|
90
|
+
raise Exception("Default account has no associated token")
|
|
96
91
|
|
|
97
|
-
|
|
92
|
+
return _generate_jwt_token(records["id"])
|