geovisio 2.9.0__py3-none-any.whl → 2.11.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- geovisio/__init__.py +8 -1
- geovisio/admin_cli/user.py +7 -2
- geovisio/config_app.py +26 -12
- geovisio/translations/ar/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/ar/LC_MESSAGES/messages.po +818 -0
- geovisio/translations/be/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/be/LC_MESSAGES/messages.po +886 -0
- geovisio/translations/br/LC_MESSAGES/messages.po +1 -1
- geovisio/translations/da/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/da/LC_MESSAGES/messages.po +96 -4
- geovisio/translations/de/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/de/LC_MESSAGES/messages.po +214 -122
- geovisio/translations/el/LC_MESSAGES/messages.po +1 -1
- geovisio/translations/en/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/en/LC_MESSAGES/messages.po +234 -157
- geovisio/translations/eo/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/eo/LC_MESSAGES/messages.po +55 -5
- geovisio/translations/es/LC_MESSAGES/messages.po +1 -1
- geovisio/translations/fi/LC_MESSAGES/messages.po +1 -1
- geovisio/translations/fr/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/fr/LC_MESSAGES/messages.po +92 -3
- geovisio/translations/hu/LC_MESSAGES/messages.po +1 -1
- geovisio/translations/it/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/it/LC_MESSAGES/messages.po +63 -3
- geovisio/translations/ja/LC_MESSAGES/messages.po +1 -1
- geovisio/translations/ko/LC_MESSAGES/messages.po +1 -1
- geovisio/translations/messages.pot +216 -139
- geovisio/translations/nl/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/nl/LC_MESSAGES/messages.po +333 -62
- geovisio/translations/oc/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/oc/LC_MESSAGES/messages.po +821 -0
- geovisio/translations/pl/LC_MESSAGES/messages.po +1 -1
- geovisio/translations/pt/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/pt/LC_MESSAGES/messages.po +944 -0
- geovisio/translations/pt_BR/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/pt_BR/LC_MESSAGES/messages.po +942 -0
- geovisio/translations/sv/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/sv/LC_MESSAGES/messages.po +4 -3
- geovisio/translations/ti/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/ti/LC_MESSAGES/messages.po +762 -0
- geovisio/translations/tr/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/tr/LC_MESSAGES/messages.po +927 -0
- geovisio/translations/uk/LC_MESSAGES/messages.mo +0 -0
- geovisio/translations/uk/LC_MESSAGES/messages.po +920 -0
- geovisio/translations/zh_Hant/LC_MESSAGES/messages.po +1 -1
- geovisio/utils/annotations.py +21 -21
- geovisio/utils/auth.py +47 -13
- geovisio/utils/cql2.py +22 -5
- geovisio/utils/fields.py +14 -2
- geovisio/utils/items.py +44 -0
- geovisio/utils/model_query.py +2 -2
- geovisio/utils/pic_shape.py +1 -1
- geovisio/utils/pictures.py +127 -36
- geovisio/utils/semantics.py +32 -3
- geovisio/utils/sentry.py +1 -1
- geovisio/utils/sequences.py +155 -109
- geovisio/utils/upload_set.py +303 -206
- geovisio/utils/users.py +18 -0
- geovisio/utils/website.py +1 -1
- geovisio/web/annotations.py +303 -69
- geovisio/web/auth.py +1 -1
- geovisio/web/collections.py +194 -97
- geovisio/web/configuration.py +36 -4
- geovisio/web/docs.py +109 -13
- geovisio/web/items.py +319 -186
- geovisio/web/map.py +92 -54
- geovisio/web/pages.py +48 -4
- geovisio/web/params.py +100 -42
- geovisio/web/pictures.py +37 -3
- geovisio/web/prepare.py +4 -2
- geovisio/web/queryables.py +57 -0
- geovisio/web/stac.py +8 -2
- geovisio/web/tokens.py +49 -1
- geovisio/web/upload_set.py +226 -51
- geovisio/web/users.py +89 -8
- geovisio/web/utils.py +26 -8
- geovisio/workers/runner_pictures.py +128 -23
- {geovisio-2.9.0.dist-info → geovisio-2.11.0.dist-info}/METADATA +15 -14
- geovisio-2.11.0.dist-info/RECORD +117 -0
- geovisio-2.9.0.dist-info/RECORD +0 -98
- {geovisio-2.9.0.dist-info → geovisio-2.11.0.dist-info}/WHEEL +0 -0
- {geovisio-2.9.0.dist-info → geovisio-2.11.0.dist-info}/licenses/LICENSE +0 -0
geovisio/web/users.py
CHANGED
|
@@ -3,6 +3,7 @@ from uuid import UUID
|
|
|
3
3
|
import flask
|
|
4
4
|
from flask import request, current_app, session, url_for
|
|
5
5
|
from flask_babel import gettext as _
|
|
6
|
+
from dataclasses import dataclass
|
|
6
7
|
from pydantic import BaseModel, ConfigDict, ValidationError, computed_field
|
|
7
8
|
from geovisio.utils import auth, db
|
|
8
9
|
from geovisio import errors
|
|
@@ -15,6 +16,7 @@ from geovisio.utils.params import validation_error
|
|
|
15
16
|
from geovisio.web import stac
|
|
16
17
|
from geovisio.web.auth import NEXT_URL_KEY
|
|
17
18
|
from geovisio.web.utils import get_root_link
|
|
19
|
+
from geovisio.web.params import Visibility, check_visibility
|
|
18
20
|
|
|
19
21
|
bp = flask.Blueprint("user", __name__, url_prefix="/api/users")
|
|
20
22
|
|
|
@@ -45,10 +47,17 @@ class UserInfo(BaseModel):
|
|
|
45
47
|
tos_accepted: Optional[bool] = None
|
|
46
48
|
"""True means the user has accepted the terms of service (tos). Can only be seen by the user itself"""
|
|
47
49
|
|
|
50
|
+
tos_latest_change_read: Optional[bool] = None
|
|
51
|
+
"""True means the user has read the latest changes to the terms of service (tos). Can only be seen by the user itself"""
|
|
52
|
+
|
|
48
53
|
permissions: Optional[Permissions] = None
|
|
49
54
|
"""The user role and permissions. Can only be seen by the user itself"""
|
|
50
55
|
|
|
51
|
-
|
|
56
|
+
default_visibility: Optional[Visibility] = None
|
|
57
|
+
"""Default visibility for all upload_sets/sequences/pictures of the user. The visibility can be overriden at the upload_set/sequence/picture level.
|
|
58
|
+
If not set, the default visibility of the instance will be used."""
|
|
59
|
+
|
|
60
|
+
model_config = ConfigDict(use_attribute_docstrings=True, use_enum_values=True)
|
|
52
61
|
|
|
53
62
|
@computed_field
|
|
54
63
|
@property
|
|
@@ -71,13 +80,43 @@ class UserInfo(BaseModel):
|
|
|
71
80
|
]
|
|
72
81
|
|
|
73
82
|
|
|
83
|
+
@dataclass
|
|
84
|
+
class AdditionalUserInfo:
|
|
85
|
+
default_visibility: Visibility
|
|
86
|
+
tos_latest_change_read: bool
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def get_additional_user_info(account: auth.Account) -> AdditionalUserInfo:
|
|
90
|
+
"""Get additional information about the user, like the default visibility"""
|
|
91
|
+
u = db.fetchone(
|
|
92
|
+
current_app,
|
|
93
|
+
"""SELECT
|
|
94
|
+
COALESCE(default_visibility, (SELECT default_visibility FROM configurations LIMIT 1)) AS default_visibility,
|
|
95
|
+
CASE WHEN
|
|
96
|
+
tos_latest_change_read_at IS NULL THEN FALSE
|
|
97
|
+
ELSE COALESCE(tos_latest_change_read_at >= (SELECT MAX(updated_at) FROM pages WHERE name = 'terms-of-service'), TRUE)
|
|
98
|
+
END AS tos_latest_change_read
|
|
99
|
+
FROM accounts WHERE id = %s""",
|
|
100
|
+
[account.id],
|
|
101
|
+
row_factory=dict_row,
|
|
102
|
+
)
|
|
103
|
+
return AdditionalUserInfo(default_visibility=u["default_visibility"], tos_latest_change_read=u["tos_latest_change_read"])
|
|
104
|
+
|
|
105
|
+
|
|
74
106
|
def _get_user_info(account: auth.Account):
|
|
75
107
|
user_info = UserInfo(id=account.id, name=account.name, collaborative_metadata=account.collaborative_metadata)
|
|
76
108
|
logged_account = auth.get_current_account()
|
|
77
|
-
if logged_account is not None and account.id == logged_account.id:
|
|
109
|
+
if logged_account is not None and (account.id == logged_account.id or logged_account.can_see_all()):
|
|
78
110
|
# we show the term of service acceptance only if the user is the logged user and if ToS are mandatory
|
|
111
|
+
# we also show all fields to the admins
|
|
79
112
|
if flask.current_app.config["API_ENFORCE_TOS_ACCEPTANCE"]:
|
|
80
113
|
user_info.tos_accepted = account.tos_accepted
|
|
114
|
+
|
|
115
|
+
# same, we only show the default visibility if the user is the logged user
|
|
116
|
+
additional_info = get_additional_user_info(account)
|
|
117
|
+
user_info.default_visibility = additional_info.default_visibility
|
|
118
|
+
user_info.tos_latest_change_read = additional_info.tos_latest_change_read
|
|
119
|
+
|
|
81
120
|
user_info.permissions = Permissions(
|
|
82
121
|
role=account.role,
|
|
83
122
|
can_check_reports=account.can_check_reports(),
|
|
@@ -91,13 +130,13 @@ def _get_user_info(account: auth.Account):
|
|
|
91
130
|
@bp.route("/me")
|
|
92
131
|
@auth.login_required_with_redirect()
|
|
93
132
|
def getMyUserInfo(account):
|
|
94
|
-
"""Get current logged user
|
|
133
|
+
"""Get current logged user information
|
|
95
134
|
---
|
|
96
135
|
tags:
|
|
97
136
|
- Users
|
|
98
137
|
responses:
|
|
99
138
|
200:
|
|
100
|
-
description: Information about the logged account
|
|
139
|
+
description: Information about the logged in account
|
|
101
140
|
content:
|
|
102
141
|
application/json:
|
|
103
142
|
schema:
|
|
@@ -108,7 +147,7 @@ def getMyUserInfo(account):
|
|
|
108
147
|
|
|
109
148
|
@bp.route("/<uuid:userId>")
|
|
110
149
|
def getUserInfo(userId):
|
|
111
|
-
"""Get user
|
|
150
|
+
"""Get user information
|
|
112
151
|
---
|
|
113
152
|
tags:
|
|
114
153
|
- Users
|
|
@@ -152,7 +191,7 @@ def getMyCatalog(account):
|
|
|
152
191
|
deprecated: true
|
|
153
192
|
responses:
|
|
154
193
|
200:
|
|
155
|
-
description: the Catalog listing all sequences associated to given user. Note that it's similar to the user's
|
|
194
|
+
description: the Catalog listing all sequences associated to given user. Note that it's similar to the user's collection, but with less metadata since a STAC collection is an enhanced STAC catalog.
|
|
156
195
|
content:
|
|
157
196
|
application/json:
|
|
158
197
|
schema:
|
|
@@ -350,9 +389,22 @@ class UserConfiguration(BaseModel):
|
|
|
350
389
|
|
|
351
390
|
If not set, it will default to the instance default collaborative editing policy."""
|
|
352
391
|
|
|
392
|
+
default_visibility: Optional[Visibility] = None
|
|
393
|
+
"""Default visibility for all upload_sets/sequences/pictures of the user. The visibility can be overriden at the upload_set/sequence/picture level.
|
|
394
|
+
If not set, the default visibility of the instance will be used."""
|
|
395
|
+
|
|
353
396
|
def has_override(self) -> bool:
|
|
354
397
|
return bool(self.model_fields_set)
|
|
355
398
|
|
|
399
|
+
def validate(self):
|
|
400
|
+
if self.default_visibility and not check_visibility(self.default_visibility):
|
|
401
|
+
raise errors.InvalidAPIUsage(
|
|
402
|
+
_("The logged-only visibility is not allowed on this instance since anybody can create an account"),
|
|
403
|
+
status_code=400,
|
|
404
|
+
)
|
|
405
|
+
|
|
406
|
+
model_config = ConfigDict(use_enum_values=True, use_attribute_docstrings=True)
|
|
407
|
+
|
|
356
408
|
|
|
357
409
|
@bp.route("/me", methods=["PATCH"])
|
|
358
410
|
@auth.login_required()
|
|
@@ -387,8 +439,11 @@ def patchUserConfiguration(account):
|
|
|
387
439
|
|
|
388
440
|
if not metadata:
|
|
389
441
|
return _get_user_info(account)
|
|
390
|
-
|
|
442
|
+
|
|
443
|
+
metadata.validate()
|
|
444
|
+
|
|
391
445
|
if metadata.has_override():
|
|
446
|
+
params = get_db_params_and_values(metadata)
|
|
392
447
|
|
|
393
448
|
fields = params.fields_for_set_list()
|
|
394
449
|
|
|
@@ -421,7 +476,9 @@ def accept_tos(account: auth.Account):
|
|
|
421
476
|
# Note: accepting twice does not change the accepted_at date
|
|
422
477
|
account = db.fetchone(
|
|
423
478
|
current_app,
|
|
424
|
-
SQL(
|
|
479
|
+
SQL(
|
|
480
|
+
"UPDATE accounts SET tos_accepted_at = COALESCE(tos_accepted_at, NOW()), tos_latest_change_read_at = NOW() WHERE id = %(account_id)s RETURNING *"
|
|
481
|
+
),
|
|
425
482
|
{"account_id": account.id},
|
|
426
483
|
row_factory=class_row(auth.Account),
|
|
427
484
|
)
|
|
@@ -431,3 +488,27 @@ def accept_tos(account: auth.Account):
|
|
|
431
488
|
session.permanent = True
|
|
432
489
|
|
|
433
490
|
return _get_user_info(account)
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
@bp.route("/me/tos_read", methods=["POST"])
|
|
494
|
+
@auth.login_required()
|
|
495
|
+
def tos_read(account: auth.Account):
|
|
496
|
+
"""
|
|
497
|
+
Mark the new terms of service changes as read.
|
|
498
|
+
---
|
|
499
|
+
tags:
|
|
500
|
+
- Auth
|
|
501
|
+
responses:
|
|
502
|
+
200:
|
|
503
|
+
description: Successfully marked the terms of service as read
|
|
504
|
+
content:
|
|
505
|
+
application/json: {}
|
|
506
|
+
"""
|
|
507
|
+
account = db.fetchone(
|
|
508
|
+
current_app,
|
|
509
|
+
SQL("UPDATE accounts SET tos_latest_change_read_at = NOW() WHERE id = %(account_id)s RETURNING *"),
|
|
510
|
+
{"account_id": account.id},
|
|
511
|
+
row_factory=class_row(auth.Account),
|
|
512
|
+
)
|
|
513
|
+
|
|
514
|
+
return "", 200
|
geovisio/web/utils.py
CHANGED
|
@@ -7,6 +7,7 @@ from geovisio import errors
|
|
|
7
7
|
from geovisio.utils import db
|
|
8
8
|
from flask import current_app, url_for
|
|
9
9
|
from flask_babel import gettext as _
|
|
10
|
+
from psycopg.rows import dict_row
|
|
10
11
|
from geovisio import __version__
|
|
11
12
|
import subprocess
|
|
12
13
|
|
|
@@ -14,12 +15,12 @@ STAC_VERSION = "1.0.0"
|
|
|
14
15
|
|
|
15
16
|
|
|
16
17
|
def removeNoneInDict(val):
|
|
17
|
-
"""Removes empty values from
|
|
18
|
+
"""Removes empty values from dictionary"""
|
|
18
19
|
return {k: v for k, v in val.items() if v is not None}
|
|
19
20
|
|
|
20
21
|
|
|
21
22
|
def cleanNoneInDict(val):
|
|
22
|
-
"""Removes empty values from
|
|
23
|
+
"""Removes empty values from dictionary, and return None if dict is empty"""
|
|
23
24
|
res = removeNoneInDict(val)
|
|
24
25
|
return res if len(res) > 0 else None
|
|
25
26
|
|
|
@@ -42,14 +43,31 @@ def cleanNoneInList(val: typing.List) -> typing.List:
|
|
|
42
43
|
return list(filter(lambda e: e is not None, val))
|
|
43
44
|
|
|
44
45
|
|
|
45
|
-
def
|
|
46
|
-
|
|
46
|
+
def get_default_account():
|
|
47
|
+
from geovisio.utils import auth
|
|
48
|
+
|
|
49
|
+
r = db.fetchone(current_app, "SELECT id, name, role FROM accounts WHERE is_default", row_factory=dict_row)
|
|
50
|
+
if not r:
|
|
51
|
+
return None
|
|
52
|
+
return auth.Account(
|
|
53
|
+
id=r["id"],
|
|
54
|
+
name=r["name"],
|
|
55
|
+
role=auth.AccountRole(r["role"]),
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def accountOrDefault(account):
|
|
60
|
+
# Get default account
|
|
47
61
|
if account is not None:
|
|
48
|
-
return account
|
|
49
|
-
|
|
50
|
-
|
|
62
|
+
return account
|
|
63
|
+
if current_app.config["API_FORCE_AUTH_ON_UPLOAD"]:
|
|
64
|
+
# if the API forces login on upload, we do not return the default account
|
|
65
|
+
return None
|
|
66
|
+
# if the API authorizes anonymous upload, we get the default account ID
|
|
67
|
+
def_account = get_default_account()
|
|
68
|
+
if def_account is None:
|
|
51
69
|
raise errors.InternalError(_("No default account defined, please contact your instance administrator"))
|
|
52
|
-
return
|
|
70
|
+
return def_account
|
|
53
71
|
|
|
54
72
|
|
|
55
73
|
def get_license_link():
|
|
@@ -17,7 +17,7 @@ from typing import Any, Dict, Optional
|
|
|
17
17
|
import threading
|
|
18
18
|
from uuid import UUID
|
|
19
19
|
from croniter import croniter
|
|
20
|
-
from datetime import datetime, timezone
|
|
20
|
+
from datetime import datetime, timezone, timedelta
|
|
21
21
|
import geovisio.utils.filesystems
|
|
22
22
|
|
|
23
23
|
log = logging.getLogger("geovisio.runner_pictures")
|
|
@@ -58,6 +58,7 @@ class ProcessTask(str, Enum):
|
|
|
58
58
|
delete = "delete"
|
|
59
59
|
dispatch = "dispatch"
|
|
60
60
|
finalize = "finalize"
|
|
61
|
+
read_metadata = "read_metadata"
|
|
61
62
|
|
|
62
63
|
|
|
63
64
|
@dataclass
|
|
@@ -65,6 +66,7 @@ class DbPicture:
|
|
|
65
66
|
id: UUID
|
|
66
67
|
metadata: dict
|
|
67
68
|
skip_blurring: bool
|
|
69
|
+
orientation: str
|
|
68
70
|
|
|
69
71
|
def blurred_by_author(self):
|
|
70
72
|
return self.metadata.get("blurredByAuthor", False)
|
|
@@ -90,6 +92,8 @@ class DbJob:
|
|
|
90
92
|
seq: Optional[DbSequence]
|
|
91
93
|
|
|
92
94
|
task: ProcessTask
|
|
95
|
+
args: Optional[Dict[Any, Any]] = None
|
|
96
|
+
warning: Optional[str] = None
|
|
93
97
|
|
|
94
98
|
def label(self):
|
|
95
99
|
impacted_object = ""
|
|
@@ -105,7 +109,7 @@ class DbJob:
|
|
|
105
109
|
return f"{self.task} for {impacted_object}"
|
|
106
110
|
|
|
107
111
|
|
|
108
|
-
def store_detection_semantics(
|
|
112
|
+
def store_detection_semantics(job: DbJob, metadata: Dict[str, Any], store_id: bool):
|
|
109
113
|
"""store the detection returned by the blurring API in the database.
|
|
110
114
|
|
|
111
115
|
The semantics part is stored as annotations, linked to the default account.
|
|
@@ -120,13 +124,13 @@ def store_detection_semantics(pic: DbPicture, metadata: Dict[str, Any], store_id
|
|
|
120
124
|
|
|
121
125
|
tags = metadata.pop("annotations", [])
|
|
122
126
|
|
|
123
|
-
with
|
|
127
|
+
with job.reporting_conn.cursor() as cursor:
|
|
124
128
|
blurring_id = metadata.get("blurring_id")
|
|
125
129
|
if blurring_id and store_id:
|
|
126
130
|
# we store the blurring id to be able to unblur the picture later
|
|
127
131
|
cursor.execute(
|
|
128
132
|
"UPDATE pictures SET blurring_id = %(blurring_id)s WHERE id = %(id)s",
|
|
129
|
-
{"blurring_id": blurring_id, "id": pic.id},
|
|
133
|
+
{"blurring_id": blurring_id, "id": job.pic.id},
|
|
130
134
|
)
|
|
131
135
|
|
|
132
136
|
if not tags:
|
|
@@ -140,21 +144,48 @@ def store_detection_semantics(pic: DbPicture, metadata: Dict[str, Any], store_id
|
|
|
140
144
|
# we want to remove all the tags added by the same bluring api previously
|
|
141
145
|
# it's especially usefull when a picture is blurred multiple times
|
|
142
146
|
# and if the detection model has been updated between the blurrings
|
|
143
|
-
semantics.delete_annotation_tags_from_service(
|
|
147
|
+
semantics.delete_annotation_tags_from_service(job.reporting_conn, job.pic.id, service_name="SGBlur", account=default_account_id)
|
|
144
148
|
try:
|
|
145
149
|
annotations_to_create = [
|
|
146
|
-
annotations.AnnotationCreationParameter(**t, account_id=default_account_id, picture_id=pic.id) for t in tags
|
|
150
|
+
annotations.AnnotationCreationParameter(**t, account_id=default_account_id, picture_id=job.pic.id) for t in tags
|
|
147
151
|
]
|
|
152
|
+
for a in annotations_to_create:
|
|
153
|
+
annotations.creation_annotation(a, job.reporting_conn)
|
|
148
154
|
except Exception as e:
|
|
149
155
|
# if the detections are not in the correct format, we skip them
|
|
150
156
|
msg = errors.getMessageFromException(e)
|
|
151
|
-
|
|
157
|
+
if hasattr(e, "payload"):
|
|
158
|
+
msg += f": {e.payload}"
|
|
159
|
+
log.error(f"impossible to save blurring detections, skipping it for picture {job.pic.id}: {msg}")
|
|
160
|
+
job.warning = msg
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def update_picture_orientation(conn: psycopg.Connection, db_pic: DbPicture, picturePillow: Image):
|
|
164
|
+
"""if the picture is side oriented, we need to check if the blurring API has rotated the picture, and update its size"""
|
|
165
|
+
if db_pic.orientation not in ("6", "8"):
|
|
152
166
|
return
|
|
153
|
-
|
|
154
|
-
|
|
167
|
+
|
|
168
|
+
new_size = utils.pictures.getPictureSizing(picturePillow)
|
|
169
|
+
if new_size["width"] != db_pic.metadata["width"] or new_size["height"] != db_pic.metadata["height"]:
|
|
170
|
+
with conn.cursor() as cursor:
|
|
171
|
+
# update the new X/Y dimensions and reset the orientation, to tell that it's no longer side oriented
|
|
172
|
+
cursor.execute(
|
|
173
|
+
"""UPDATE pictures SET
|
|
174
|
+
exif = exif - 'Exif.Image.Orientation' || jsonb_build_object('Exif.Photo.PixelXDimension', %(width)s, 'Exif.Photo.PixelYDimension', %(height)s),
|
|
175
|
+
metadata = metadata || jsonb_build_object('width', %(width)s, 'height', %(height)s, 'cols', %(cols)s, 'rows', %(rows)s)
|
|
176
|
+
WHERE id = %(id)s
|
|
177
|
+
""",
|
|
178
|
+
{
|
|
179
|
+
"width": new_size["width"],
|
|
180
|
+
"height": new_size["height"],
|
|
181
|
+
"id": db_pic.id,
|
|
182
|
+
"cols": new_size["cols"],
|
|
183
|
+
"rows": new_size["rows"],
|
|
184
|
+
},
|
|
185
|
+
)
|
|
155
186
|
|
|
156
187
|
|
|
157
|
-
def processPictureFiles(
|
|
188
|
+
def processPictureFiles(job: DbJob, config):
|
|
158
189
|
"""Generates the files associated with a sequence picture.
|
|
159
190
|
|
|
160
191
|
If needed the image is blurred before the tiles and thumbnail are generated.
|
|
@@ -168,6 +199,7 @@ def processPictureFiles(pic: DbPicture, config):
|
|
|
168
199
|
config : dict
|
|
169
200
|
Flask app.config (passed as param to allow using ThreadPoolExecutor)
|
|
170
201
|
"""
|
|
202
|
+
pic = job.pic
|
|
171
203
|
skipBlur = pic.skip_blurring or config.get("API_BLUR_URL") is None
|
|
172
204
|
fses = config["FILESYSTEMS"]
|
|
173
205
|
fs = fses.permanent if skipBlur else fses.tmp
|
|
@@ -206,8 +238,12 @@ def processPictureFiles(pic: DbPicture, config):
|
|
|
206
238
|
picture = None
|
|
207
239
|
else:
|
|
208
240
|
picture = res.image
|
|
241
|
+
|
|
242
|
+
if pic.orientation in ("6", "8"):
|
|
243
|
+
update_picture_orientation(job.reporting_conn, pic, picture)
|
|
244
|
+
|
|
209
245
|
if res.metadata:
|
|
210
|
-
store_detection_semantics(
|
|
246
|
+
store_detection_semantics(job, res.metadata, store_id=config["PICTURE_PROCESS_KEEP_UNBLURRED_PARTS"])
|
|
211
247
|
|
|
212
248
|
# Delete original unblurred file
|
|
213
249
|
geovisio.utils.filesystems.removeFsEvenNotFound(fses.tmp, picHdPath)
|
|
@@ -287,7 +323,9 @@ class PictureProcessor:
|
|
|
287
323
|
if self.app.pool.closed and self.stop:
|
|
288
324
|
# in some tests, the pool is closed before the worker is stopped, we check this here
|
|
289
325
|
return
|
|
290
|
-
self.
|
|
326
|
+
if not self.stop:
|
|
327
|
+
# periodic tasks are only checked by permanent workers
|
|
328
|
+
self.check_periodic_tasks()
|
|
291
329
|
r = process_next_job(self.app)
|
|
292
330
|
if not r:
|
|
293
331
|
if self.stop:
|
|
@@ -308,7 +346,7 @@ class PictureProcessor:
|
|
|
308
346
|
signal.signal(signal.SIGTERM, self._graceful_shutdown)
|
|
309
347
|
|
|
310
348
|
def _graceful_shutdown(self, *args):
|
|
311
|
-
log.info("
|
|
349
|
+
log.info("Stopping worker, waiting for last picture processing to finish...")
|
|
312
350
|
self.stop = True
|
|
313
351
|
|
|
314
352
|
def check_periodic_tasks(self):
|
|
@@ -364,16 +402,19 @@ def process_next_job(app):
|
|
|
364
402
|
span.set_data("pic_id", job.pic.id)
|
|
365
403
|
with utils.time.log_elapsed(f"Processing picture {job.pic.id}"):
|
|
366
404
|
# open another connection for reporting and queries
|
|
367
|
-
processPictureFiles(job
|
|
405
|
+
processPictureFiles(job, app.config)
|
|
368
406
|
elif job.task == ProcessTask.delete and job.pic:
|
|
369
407
|
with sentry_sdk.start_span(description="Deleting picture") as span:
|
|
370
408
|
span.set_data("pic_id", job.pic.id)
|
|
371
409
|
with utils.time.log_elapsed(f"Deleting picture {job.pic.id}"):
|
|
372
|
-
_delete_picture(job.pic)
|
|
410
|
+
_delete_picture(job.reporting_conn, job.pic)
|
|
411
|
+
elif job.task == ProcessTask.read_metadata and job.pic:
|
|
412
|
+
with utils.time.log_elapsed(f"Reading metadata of picture {job.pic.id}"):
|
|
413
|
+
_read_picture_metadata(job.pic, **(job.args or {}))
|
|
373
414
|
elif job.task == ProcessTask.dispatch and job.upload_set:
|
|
374
415
|
with utils.time.log_elapsed(f"Dispatching upload set {job.upload_set.id}"):
|
|
375
416
|
try:
|
|
376
|
-
upload_set.dispatch(job.upload_set.id)
|
|
417
|
+
upload_set.dispatch(job.reporting_conn, job.upload_set.id)
|
|
377
418
|
except Exception as e:
|
|
378
419
|
log.exception(f"impossible to dispatch upload set {job.upload_set.id}")
|
|
379
420
|
raise RecoverableProcessException("Upload set dispatch error: " + errors.getMessageFromException(e)) from e
|
|
@@ -399,7 +440,7 @@ def _get_next_job(app):
|
|
|
399
440
|
with app.pool.connection() as locking_transaction:
|
|
400
441
|
with locking_transaction.transaction(), locking_transaction.cursor(row_factory=dict_row) as cursor:
|
|
401
442
|
r = cursor.execute(
|
|
402
|
-
"""SELECT j.id, j.picture_id, j.upload_set_id, j.sequence_id, j.task, j.picture_to_delete_id, p.metadata, j.args
|
|
443
|
+
"""SELECT j.id, j.picture_id, j.upload_set_id, j.sequence_id, j.task, j.picture_to_delete_id, p.metadata, j.args, p.exif->'Exif.Image.Orientation' as orientation
|
|
403
444
|
FROM job_queue j
|
|
404
445
|
LEFT JOIN pictures p ON p.id = j.picture_id
|
|
405
446
|
ORDER by
|
|
@@ -418,7 +459,12 @@ def _get_next_job(app):
|
|
|
418
459
|
# (and it will not a foreign key since the picture's row will already have been deleted from the db)
|
|
419
460
|
pic_id = r["picture_id"] or r["picture_to_delete_id"]
|
|
420
461
|
db_pic = (
|
|
421
|
-
DbPicture(
|
|
462
|
+
DbPicture(
|
|
463
|
+
id=pic_id,
|
|
464
|
+
metadata=r["metadata"],
|
|
465
|
+
skip_blurring=(r["args"] or {}).get("skip_blurring", False),
|
|
466
|
+
orientation=r["orientation"],
|
|
467
|
+
)
|
|
422
468
|
if pic_id is not None
|
|
423
469
|
else None
|
|
424
470
|
)
|
|
@@ -466,7 +512,7 @@ def _get_next_job(app):
|
|
|
466
512
|
_finalize_sequence(job)
|
|
467
513
|
error = e
|
|
468
514
|
|
|
469
|
-
# we raise an error after the transaction has been
|
|
515
|
+
# we raise an error after the transaction has been committed to be sure to have the state persisted in the database
|
|
470
516
|
if error:
|
|
471
517
|
raise error
|
|
472
518
|
|
|
@@ -507,9 +553,16 @@ def _finalize_job(conn, job: DbJob):
|
|
|
507
553
|
f"The job {job.job_history_id} ({job.label()}) has likely been deleted during the process (it can happen if the picture/upload_set/sequence has been deleted by another process), we don't need to finalize it"
|
|
508
554
|
)
|
|
509
555
|
return
|
|
556
|
+
|
|
557
|
+
params = {"id": job.job_history_id}
|
|
558
|
+
fields = [SQL("finished_at = CURRENT_TIMESTAMP")]
|
|
559
|
+
if job.warning:
|
|
560
|
+
fields.append(SQL("warning = %(warn)s"))
|
|
561
|
+
params["warn"] = job.warning
|
|
562
|
+
|
|
510
563
|
job.reporting_conn.execute(
|
|
511
|
-
"UPDATE job_history SET
|
|
512
|
-
|
|
564
|
+
SQL("UPDATE job_history SET {fields} WHERE id = %(id)s").format(fields=SQL(", ").join(fields)),
|
|
565
|
+
params,
|
|
513
566
|
)
|
|
514
567
|
if job.task == ProcessTask.prepare and job.pic:
|
|
515
568
|
# Note: the status is slowly been deprecated by replacing it with more precise status, and in the end it will be removed
|
|
@@ -544,7 +597,7 @@ def _initialize_job(
|
|
|
544
597
|
"pic_to_delete": db_pic.id if db_pic and task == ProcessTask.delete else None,
|
|
545
598
|
"us_id": db_upload_set.id if db_upload_set else None,
|
|
546
599
|
"task": task.value,
|
|
547
|
-
"args": Jsonb(args),
|
|
600
|
+
"args": Jsonb(args) if args else None,
|
|
548
601
|
},
|
|
549
602
|
).fetchone()
|
|
550
603
|
|
|
@@ -559,6 +612,7 @@ def _initialize_job(
|
|
|
559
612
|
upload_set=db_upload_set,
|
|
560
613
|
task=task,
|
|
561
614
|
job_history_id=r[0],
|
|
615
|
+
args=args,
|
|
562
616
|
)
|
|
563
617
|
|
|
564
618
|
|
|
@@ -608,7 +662,58 @@ def _mark_process_as_error(
|
|
|
608
662
|
conn.execute("DELETE FROM job_queue WHERE id = %(id)s", {"id": job.job_queue_id})
|
|
609
663
|
|
|
610
664
|
|
|
611
|
-
def _delete_picture(pic: DbPicture):
|
|
665
|
+
def _delete_picture(conn: psycopg.Connection, pic: DbPicture):
|
|
612
666
|
"""Delete a picture from the filesystem"""
|
|
613
667
|
log.debug(f"Deleting picture files {pic.id}")
|
|
668
|
+
|
|
669
|
+
def check_if_no_workers_preparing():
|
|
670
|
+
try:
|
|
671
|
+
# We try to check if there at some workers preparing this picture, if it's the case, we wait a bit and retry.
|
|
672
|
+
# after some time, if the lock is still not released, we raise a RetryLaterProcessException, to reschedule the whole job later
|
|
673
|
+
conn.execute(
|
|
674
|
+
"SELECT id FROM job_queue WHERE picture_id = %(id)s and task = 'prepare' FOR UPDATE NOWAIT",
|
|
675
|
+
{"id": pic.id},
|
|
676
|
+
)
|
|
677
|
+
return True
|
|
678
|
+
except psycopg.errors.LockNotAvailable:
|
|
679
|
+
logging.debug(f"The picture {pic.id} is being processed, we'll retry later")
|
|
680
|
+
return False
|
|
681
|
+
|
|
682
|
+
_retry_for(check_if_no_workers_preparing, error=f"Picture {pic.id} is being processed")
|
|
683
|
+
|
|
684
|
+
# Delete the row if needed (note that it can have already been deleted (for example if a whole upload_set has been deleted, the `ON DELETE CASCADE` deletes all the pictures's row (but the files still need to be deleted)))
|
|
685
|
+
conn.execute("DELETE FROM pictures WHERE id = %(id)s", {"id": pic.id})
|
|
686
|
+
|
|
614
687
|
utils.pictures.removeAllFiles(pic.id)
|
|
688
|
+
|
|
689
|
+
|
|
690
|
+
def _retry_for(func, error, timeout=timedelta(minutes=1), sleep=timedelta(seconds=5)):
|
|
691
|
+
import time
|
|
692
|
+
|
|
693
|
+
cur_duration = timedelta(seconds=0)
|
|
694
|
+
while cur_duration < timeout:
|
|
695
|
+
r = func()
|
|
696
|
+
if r:
|
|
697
|
+
return
|
|
698
|
+
cur_duration += sleep
|
|
699
|
+
time.sleep(sleep.total_seconds())
|
|
700
|
+
|
|
701
|
+
raise RetryLaterProcessException(error)
|
|
702
|
+
|
|
703
|
+
|
|
704
|
+
def _read_picture_metadata(picture: DbPicture, read_file=False):
|
|
705
|
+
"""Reread the picture metadata.
|
|
706
|
+
|
|
707
|
+
Normally the picture's metadata are read during upload, but sometimes (mainly when the geopic-tag-reader library has been updated),
|
|
708
|
+
we need to read the metadata again.
|
|
709
|
+
|
|
710
|
+
Parameters
|
|
711
|
+
----------
|
|
712
|
+
picture_id : UUID
|
|
713
|
+
The ID of the picture to read the metadata from
|
|
714
|
+
read_file : bool
|
|
715
|
+
If True, the picture's raw metadata will be read again, else the Exif tools stored in the database will be used (way faster).
|
|
716
|
+
"""
|
|
717
|
+
|
|
718
|
+
with db.conn(current_app) as conn:
|
|
719
|
+
utils.pictures.update_picture_metadata(conn, picture.id, read_file)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: geovisio
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.11.0
|
|
4
4
|
Summary: GeoVisio API - Main
|
|
5
5
|
Author-email: Adrien PAVIE <panieravide@riseup.net>, Antoine Desbordes <antoine.desbordes@gmail.com>
|
|
6
6
|
Requires-Python: >=3.10
|
|
@@ -11,7 +11,7 @@ Requires-Dist: Flask ~= 3.1
|
|
|
11
11
|
Requires-Dist: psycopg[pool] ~= 3.2
|
|
12
12
|
Requires-Dist: flasgger ~= 0.9.7
|
|
13
13
|
Requires-Dist: Pillow ~= 11.1
|
|
14
|
-
Requires-Dist: Flask-Cors ~=
|
|
14
|
+
Requires-Dist: Flask-Cors ~= 6.0
|
|
15
15
|
Requires-Dist: fs ~= 2.4
|
|
16
16
|
Requires-Dist: fs-s3fs-forked ~= 1.1.4
|
|
17
17
|
Requires-Dist: flask-compress ~= 1.14
|
|
@@ -19,38 +19,39 @@ Requires-Dist: requests ~= 2.31
|
|
|
19
19
|
Requires-Dist: yoyo-migrations ~= 9.0
|
|
20
20
|
Requires-Dist: psycopg-binary ~= 3.2
|
|
21
21
|
Requires-Dist: python-dotenv ~= 1.1
|
|
22
|
-
Requires-Dist: authlib ~= 1.
|
|
22
|
+
Requires-Dist: authlib ~= 1.6
|
|
23
23
|
Requires-Dist: Flask-Executor ~= 1.0
|
|
24
|
-
Requires-Dist: geopic-tag-reader[write-exif]
|
|
24
|
+
Requires-Dist: geopic-tag-reader[write-exif] ~= 1.8.0
|
|
25
25
|
Requires-Dist: rfeed ~= 1.1.1
|
|
26
26
|
Requires-Dist: sentry-sdk[flask] ~= 2.24
|
|
27
27
|
Requires-Dist: pygeofilter[backend-native] ~= 0.3.1
|
|
28
28
|
Requires-Dist: python-dateutil ~= 2.9.0
|
|
29
29
|
Requires-Dist: tzdata ~= 2025.2
|
|
30
30
|
Requires-Dist: croniter ~= 6.0.0
|
|
31
|
-
Requires-Dist: pydantic ~= 2.
|
|
31
|
+
Requires-Dist: pydantic ~= 2.12
|
|
32
32
|
Requires-Dist: pydantic-extra-types ~= 2.7
|
|
33
33
|
Requires-Dist: flask-babel ~= 4.0.0
|
|
34
|
-
Requires-Dist: geojson-pydantic ~=
|
|
34
|
+
Requires-Dist: geojson-pydantic ~= 2.0.0
|
|
35
35
|
Requires-Dist: email-validator ~= 2.2.0
|
|
36
36
|
Requires-Dist: multipart>=1.2.1
|
|
37
|
+
Requires-Dist: gunicorn ~= 23.0.0
|
|
37
38
|
Requires-Dist: flit ~= 3.9.0 ; extra == "build"
|
|
38
|
-
Requires-Dist: coverage ~= 7.
|
|
39
|
-
Requires-Dist: protobuf ~=
|
|
40
|
-
Requires-Dist: mapbox-vector-tile ~= 2.
|
|
39
|
+
Requires-Dist: coverage ~= 7.9 ; extra == "dev"
|
|
40
|
+
Requires-Dist: protobuf ~= 5.26 ; extra == "dev"
|
|
41
|
+
Requires-Dist: mapbox-vector-tile ~= 2.1 ; extra == "dev"
|
|
41
42
|
Requires-Dist: pystac ~= 1.9 ; extra == "dev"
|
|
42
|
-
Requires-Dist: pytest ~= 8.
|
|
43
|
-
Requires-Dist: pytest-datafiles ~=
|
|
43
|
+
Requires-Dist: pytest ~= 8.4 ; extra == "dev"
|
|
44
|
+
Requires-Dist: pytest-datafiles ~= 3.0 ; extra == "dev"
|
|
44
45
|
Requires-Dist: pyexiv2 ~= 2.15 ; extra == "dev"
|
|
45
|
-
Requires-Dist: testcontainers ~= 4.
|
|
46
|
+
Requires-Dist: testcontainers ~= 4.10 ; extra == "dev"
|
|
46
47
|
Requires-Dist: requests-mock ~= 1.11 ; extra == "dev"
|
|
47
48
|
Requires-Dist: black ~= 25.1 ; extra == "dev"
|
|
48
49
|
Requires-Dist: pre-commit ~= 4.2 ; extra == "dev"
|
|
49
50
|
Requires-Dist: pyyaml ~= 6.0 ; extra == "dev"
|
|
50
51
|
Requires-Dist: openapi-spec-validator ~= 0.7 ; extra == "dev"
|
|
51
52
|
Requires-Dist: stac-api-validator ~= 0.6.4 ; extra == "dev"
|
|
52
|
-
Requires-Dist: mkdocs-material ~= 9.6.
|
|
53
|
-
Requires-Dist: mkdocs-swagger-ui-tag ~= 0.
|
|
53
|
+
Requires-Dist: mkdocs-material ~= 9.6.14 ; extra == "docs"
|
|
54
|
+
Requires-Dist: mkdocs-swagger-ui-tag ~= 0.7.1 ; extra == "docs"
|
|
54
55
|
Project-URL: Home, https://gitlab.com/panoramax/server/api
|
|
55
56
|
Project-URL: Source Code, https://gitlab.com/panoramax/server/api
|
|
56
57
|
Provides-Extra: build
|