geovisio 2.9.0__py3-none-any.whl → 2.11.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. geovisio/__init__.py +8 -1
  2. geovisio/admin_cli/user.py +7 -2
  3. geovisio/config_app.py +26 -12
  4. geovisio/translations/ar/LC_MESSAGES/messages.mo +0 -0
  5. geovisio/translations/ar/LC_MESSAGES/messages.po +818 -0
  6. geovisio/translations/be/LC_MESSAGES/messages.mo +0 -0
  7. geovisio/translations/be/LC_MESSAGES/messages.po +886 -0
  8. geovisio/translations/br/LC_MESSAGES/messages.po +1 -1
  9. geovisio/translations/da/LC_MESSAGES/messages.mo +0 -0
  10. geovisio/translations/da/LC_MESSAGES/messages.po +96 -4
  11. geovisio/translations/de/LC_MESSAGES/messages.mo +0 -0
  12. geovisio/translations/de/LC_MESSAGES/messages.po +214 -122
  13. geovisio/translations/el/LC_MESSAGES/messages.po +1 -1
  14. geovisio/translations/en/LC_MESSAGES/messages.mo +0 -0
  15. geovisio/translations/en/LC_MESSAGES/messages.po +234 -157
  16. geovisio/translations/eo/LC_MESSAGES/messages.mo +0 -0
  17. geovisio/translations/eo/LC_MESSAGES/messages.po +55 -5
  18. geovisio/translations/es/LC_MESSAGES/messages.po +1 -1
  19. geovisio/translations/fi/LC_MESSAGES/messages.po +1 -1
  20. geovisio/translations/fr/LC_MESSAGES/messages.mo +0 -0
  21. geovisio/translations/fr/LC_MESSAGES/messages.po +92 -3
  22. geovisio/translations/hu/LC_MESSAGES/messages.po +1 -1
  23. geovisio/translations/it/LC_MESSAGES/messages.mo +0 -0
  24. geovisio/translations/it/LC_MESSAGES/messages.po +63 -3
  25. geovisio/translations/ja/LC_MESSAGES/messages.po +1 -1
  26. geovisio/translations/ko/LC_MESSAGES/messages.po +1 -1
  27. geovisio/translations/messages.pot +216 -139
  28. geovisio/translations/nl/LC_MESSAGES/messages.mo +0 -0
  29. geovisio/translations/nl/LC_MESSAGES/messages.po +333 -62
  30. geovisio/translations/oc/LC_MESSAGES/messages.mo +0 -0
  31. geovisio/translations/oc/LC_MESSAGES/messages.po +821 -0
  32. geovisio/translations/pl/LC_MESSAGES/messages.po +1 -1
  33. geovisio/translations/pt/LC_MESSAGES/messages.mo +0 -0
  34. geovisio/translations/pt/LC_MESSAGES/messages.po +944 -0
  35. geovisio/translations/pt_BR/LC_MESSAGES/messages.mo +0 -0
  36. geovisio/translations/pt_BR/LC_MESSAGES/messages.po +942 -0
  37. geovisio/translations/sv/LC_MESSAGES/messages.mo +0 -0
  38. geovisio/translations/sv/LC_MESSAGES/messages.po +4 -3
  39. geovisio/translations/ti/LC_MESSAGES/messages.mo +0 -0
  40. geovisio/translations/ti/LC_MESSAGES/messages.po +762 -0
  41. geovisio/translations/tr/LC_MESSAGES/messages.mo +0 -0
  42. geovisio/translations/tr/LC_MESSAGES/messages.po +927 -0
  43. geovisio/translations/uk/LC_MESSAGES/messages.mo +0 -0
  44. geovisio/translations/uk/LC_MESSAGES/messages.po +920 -0
  45. geovisio/translations/zh_Hant/LC_MESSAGES/messages.po +1 -1
  46. geovisio/utils/annotations.py +21 -21
  47. geovisio/utils/auth.py +47 -13
  48. geovisio/utils/cql2.py +22 -5
  49. geovisio/utils/fields.py +14 -2
  50. geovisio/utils/items.py +44 -0
  51. geovisio/utils/model_query.py +2 -2
  52. geovisio/utils/pic_shape.py +1 -1
  53. geovisio/utils/pictures.py +127 -36
  54. geovisio/utils/semantics.py +32 -3
  55. geovisio/utils/sentry.py +1 -1
  56. geovisio/utils/sequences.py +155 -109
  57. geovisio/utils/upload_set.py +303 -206
  58. geovisio/utils/users.py +18 -0
  59. geovisio/utils/website.py +1 -1
  60. geovisio/web/annotations.py +303 -69
  61. geovisio/web/auth.py +1 -1
  62. geovisio/web/collections.py +194 -97
  63. geovisio/web/configuration.py +36 -4
  64. geovisio/web/docs.py +109 -13
  65. geovisio/web/items.py +319 -186
  66. geovisio/web/map.py +92 -54
  67. geovisio/web/pages.py +48 -4
  68. geovisio/web/params.py +100 -42
  69. geovisio/web/pictures.py +37 -3
  70. geovisio/web/prepare.py +4 -2
  71. geovisio/web/queryables.py +57 -0
  72. geovisio/web/stac.py +8 -2
  73. geovisio/web/tokens.py +49 -1
  74. geovisio/web/upload_set.py +226 -51
  75. geovisio/web/users.py +89 -8
  76. geovisio/web/utils.py +26 -8
  77. geovisio/workers/runner_pictures.py +128 -23
  78. {geovisio-2.9.0.dist-info → geovisio-2.11.0.dist-info}/METADATA +15 -14
  79. geovisio-2.11.0.dist-info/RECORD +117 -0
  80. geovisio-2.9.0.dist-info/RECORD +0 -98
  81. {geovisio-2.9.0.dist-info → geovisio-2.11.0.dist-info}/WHEEL +0 -0
  82. {geovisio-2.9.0.dist-info → geovisio-2.11.0.dist-info}/licenses/LICENSE +0 -0
geovisio/web/users.py CHANGED
@@ -3,6 +3,7 @@ from uuid import UUID
3
3
  import flask
4
4
  from flask import request, current_app, session, url_for
5
5
  from flask_babel import gettext as _
6
+ from dataclasses import dataclass
6
7
  from pydantic import BaseModel, ConfigDict, ValidationError, computed_field
7
8
  from geovisio.utils import auth, db
8
9
  from geovisio import errors
@@ -15,6 +16,7 @@ from geovisio.utils.params import validation_error
15
16
  from geovisio.web import stac
16
17
  from geovisio.web.auth import NEXT_URL_KEY
17
18
  from geovisio.web.utils import get_root_link
19
+ from geovisio.web.params import Visibility, check_visibility
18
20
 
19
21
  bp = flask.Blueprint("user", __name__, url_prefix="/api/users")
20
22
 
@@ -45,10 +47,17 @@ class UserInfo(BaseModel):
45
47
  tos_accepted: Optional[bool] = None
46
48
  """True means the user has accepted the terms of service (tos). Can only be seen by the user itself"""
47
49
 
50
+ tos_latest_change_read: Optional[bool] = None
51
+ """True means the user has read the latest changes to the terms of service (tos). Can only be seen by the user itself"""
52
+
48
53
  permissions: Optional[Permissions] = None
49
54
  """The user role and permissions. Can only be seen by the user itself"""
50
55
 
51
- model_config = ConfigDict(use_attribute_docstrings=True)
56
+ default_visibility: Optional[Visibility] = None
57
+ """Default visibility for all upload_sets/sequences/pictures of the user. The visibility can be overriden at the upload_set/sequence/picture level.
58
+ If not set, the default visibility of the instance will be used."""
59
+
60
+ model_config = ConfigDict(use_attribute_docstrings=True, use_enum_values=True)
52
61
 
53
62
  @computed_field
54
63
  @property
@@ -71,13 +80,43 @@ class UserInfo(BaseModel):
71
80
  ]
72
81
 
73
82
 
83
+ @dataclass
84
+ class AdditionalUserInfo:
85
+ default_visibility: Visibility
86
+ tos_latest_change_read: bool
87
+
88
+
89
+ def get_additional_user_info(account: auth.Account) -> AdditionalUserInfo:
90
+ """Get additional information about the user, like the default visibility"""
91
+ u = db.fetchone(
92
+ current_app,
93
+ """SELECT
94
+ COALESCE(default_visibility, (SELECT default_visibility FROM configurations LIMIT 1)) AS default_visibility,
95
+ CASE WHEN
96
+ tos_latest_change_read_at IS NULL THEN FALSE
97
+ ELSE COALESCE(tos_latest_change_read_at >= (SELECT MAX(updated_at) FROM pages WHERE name = 'terms-of-service'), TRUE)
98
+ END AS tos_latest_change_read
99
+ FROM accounts WHERE id = %s""",
100
+ [account.id],
101
+ row_factory=dict_row,
102
+ )
103
+ return AdditionalUserInfo(default_visibility=u["default_visibility"], tos_latest_change_read=u["tos_latest_change_read"])
104
+
105
+
74
106
  def _get_user_info(account: auth.Account):
75
107
  user_info = UserInfo(id=account.id, name=account.name, collaborative_metadata=account.collaborative_metadata)
76
108
  logged_account = auth.get_current_account()
77
- if logged_account is not None and account.id == logged_account.id:
109
+ if logged_account is not None and (account.id == logged_account.id or logged_account.can_see_all()):
78
110
  # we show the term of service acceptance only if the user is the logged user and if ToS are mandatory
111
+ # we also show all fields to the admins
79
112
  if flask.current_app.config["API_ENFORCE_TOS_ACCEPTANCE"]:
80
113
  user_info.tos_accepted = account.tos_accepted
114
+
115
+ # same, we only show the default visibility if the user is the logged user
116
+ additional_info = get_additional_user_info(account)
117
+ user_info.default_visibility = additional_info.default_visibility
118
+ user_info.tos_latest_change_read = additional_info.tos_latest_change_read
119
+
81
120
  user_info.permissions = Permissions(
82
121
  role=account.role,
83
122
  can_check_reports=account.can_check_reports(),
@@ -91,13 +130,13 @@ def _get_user_info(account: auth.Account):
91
130
  @bp.route("/me")
92
131
  @auth.login_required_with_redirect()
93
132
  def getMyUserInfo(account):
94
- """Get current logged user informations
133
+ """Get current logged user information
95
134
  ---
96
135
  tags:
97
136
  - Users
98
137
  responses:
99
138
  200:
100
- description: Information about the logged account
139
+ description: Information about the logged in account
101
140
  content:
102
141
  application/json:
103
142
  schema:
@@ -108,7 +147,7 @@ def getMyUserInfo(account):
108
147
 
109
148
  @bp.route("/<uuid:userId>")
110
149
  def getUserInfo(userId):
111
- """Get user informations
150
+ """Get user information
112
151
  ---
113
152
  tags:
114
153
  - Users
@@ -152,7 +191,7 @@ def getMyCatalog(account):
152
191
  deprecated: true
153
192
  responses:
154
193
  200:
155
- description: the Catalog listing all sequences associated to given user. Note that it's similar to the user's colletion, but with less metadata since a STAC collection is an enhanced STAC catalog.
194
+ description: the Catalog listing all sequences associated to given user. Note that it's similar to the user's collection, but with less metadata since a STAC collection is an enhanced STAC catalog.
156
195
  content:
157
196
  application/json:
158
197
  schema:
@@ -350,9 +389,22 @@ class UserConfiguration(BaseModel):
350
389
 
351
390
  If not set, it will default to the instance default collaborative editing policy."""
352
391
 
392
+ default_visibility: Optional[Visibility] = None
393
+ """Default visibility for all upload_sets/sequences/pictures of the user. The visibility can be overriden at the upload_set/sequence/picture level.
394
+ If not set, the default visibility of the instance will be used."""
395
+
353
396
  def has_override(self) -> bool:
354
397
  return bool(self.model_fields_set)
355
398
 
399
+ def validate(self):
400
+ if self.default_visibility and not check_visibility(self.default_visibility):
401
+ raise errors.InvalidAPIUsage(
402
+ _("The logged-only visibility is not allowed on this instance since anybody can create an account"),
403
+ status_code=400,
404
+ )
405
+
406
+ model_config = ConfigDict(use_enum_values=True, use_attribute_docstrings=True)
407
+
356
408
 
357
409
  @bp.route("/me", methods=["PATCH"])
358
410
  @auth.login_required()
@@ -387,8 +439,11 @@ def patchUserConfiguration(account):
387
439
 
388
440
  if not metadata:
389
441
  return _get_user_info(account)
390
- params = get_db_params_and_values(metadata)
442
+
443
+ metadata.validate()
444
+
391
445
  if metadata.has_override():
446
+ params = get_db_params_and_values(metadata)
392
447
 
393
448
  fields = params.fields_for_set_list()
394
449
 
@@ -421,7 +476,9 @@ def accept_tos(account: auth.Account):
421
476
  # Note: accepting twice does not change the accepted_at date
422
477
  account = db.fetchone(
423
478
  current_app,
424
- SQL("UPDATE accounts SET tos_accepted_at = COALESCE(tos_accepted_at, NOW()) WHERE id = %(account_id)s RETURNING *"),
479
+ SQL(
480
+ "UPDATE accounts SET tos_accepted_at = COALESCE(tos_accepted_at, NOW()), tos_latest_change_read_at = NOW() WHERE id = %(account_id)s RETURNING *"
481
+ ),
425
482
  {"account_id": account.id},
426
483
  row_factory=class_row(auth.Account),
427
484
  )
@@ -431,3 +488,27 @@ def accept_tos(account: auth.Account):
431
488
  session.permanent = True
432
489
 
433
490
  return _get_user_info(account)
491
+
492
+
493
+ @bp.route("/me/tos_read", methods=["POST"])
494
+ @auth.login_required()
495
+ def tos_read(account: auth.Account):
496
+ """
497
+ Mark the new terms of service changes as read.
498
+ ---
499
+ tags:
500
+ - Auth
501
+ responses:
502
+ 200:
503
+ description: Successfully marked the terms of service as read
504
+ content:
505
+ application/json: {}
506
+ """
507
+ account = db.fetchone(
508
+ current_app,
509
+ SQL("UPDATE accounts SET tos_latest_change_read_at = NOW() WHERE id = %(account_id)s RETURNING *"),
510
+ {"account_id": account.id},
511
+ row_factory=class_row(auth.Account),
512
+ )
513
+
514
+ return "", 200
geovisio/web/utils.py CHANGED
@@ -7,6 +7,7 @@ from geovisio import errors
7
7
  from geovisio.utils import db
8
8
  from flask import current_app, url_for
9
9
  from flask_babel import gettext as _
10
+ from psycopg.rows import dict_row
10
11
  from geovisio import __version__
11
12
  import subprocess
12
13
 
@@ -14,12 +15,12 @@ STAC_VERSION = "1.0.0"
14
15
 
15
16
 
16
17
  def removeNoneInDict(val):
17
- """Removes empty values from dictionnary"""
18
+ """Removes empty values from dictionary"""
18
19
  return {k: v for k, v in val.items() if v is not None}
19
20
 
20
21
 
21
22
  def cleanNoneInDict(val):
22
- """Removes empty values from dictionnary, and return None if dict is empty"""
23
+ """Removes empty values from dictionary, and return None if dict is empty"""
23
24
  res = removeNoneInDict(val)
24
25
  return res if len(res) > 0 else None
25
26
 
@@ -42,14 +43,31 @@ def cleanNoneInList(val: typing.List) -> typing.List:
42
43
  return list(filter(lambda e: e is not None, val))
43
44
 
44
45
 
45
- def accountIdOrDefault(account):
46
- # Get default account ID
46
+ def get_default_account():
47
+ from geovisio.utils import auth
48
+
49
+ r = db.fetchone(current_app, "SELECT id, name, role FROM accounts WHERE is_default", row_factory=dict_row)
50
+ if not r:
51
+ return None
52
+ return auth.Account(
53
+ id=r["id"],
54
+ name=r["name"],
55
+ role=auth.AccountRole(r["role"]),
56
+ )
57
+
58
+
59
+ def accountOrDefault(account):
60
+ # Get default account
47
61
  if account is not None:
48
- return account.id
49
- accountId = db.fetchone(current_app, "SELECT id FROM accounts WHERE is_default")
50
- if accountId is None:
62
+ return account
63
+ if current_app.config["API_FORCE_AUTH_ON_UPLOAD"]:
64
+ # if the API forces login on upload, we do not return the default account
65
+ return None
66
+ # if the API authorizes anonymous upload, we get the default account ID
67
+ def_account = get_default_account()
68
+ if def_account is None:
51
69
  raise errors.InternalError(_("No default account defined, please contact your instance administrator"))
52
- return str(accountId[0])
70
+ return def_account
53
71
 
54
72
 
55
73
  def get_license_link():
@@ -17,7 +17,7 @@ from typing import Any, Dict, Optional
17
17
  import threading
18
18
  from uuid import UUID
19
19
  from croniter import croniter
20
- from datetime import datetime, timezone
20
+ from datetime import datetime, timezone, timedelta
21
21
  import geovisio.utils.filesystems
22
22
 
23
23
  log = logging.getLogger("geovisio.runner_pictures")
@@ -58,6 +58,7 @@ class ProcessTask(str, Enum):
58
58
  delete = "delete"
59
59
  dispatch = "dispatch"
60
60
  finalize = "finalize"
61
+ read_metadata = "read_metadata"
61
62
 
62
63
 
63
64
  @dataclass
@@ -65,6 +66,7 @@ class DbPicture:
65
66
  id: UUID
66
67
  metadata: dict
67
68
  skip_blurring: bool
69
+ orientation: str
68
70
 
69
71
  def blurred_by_author(self):
70
72
  return self.metadata.get("blurredByAuthor", False)
@@ -90,6 +92,8 @@ class DbJob:
90
92
  seq: Optional[DbSequence]
91
93
 
92
94
  task: ProcessTask
95
+ args: Optional[Dict[Any, Any]] = None
96
+ warning: Optional[str] = None
93
97
 
94
98
  def label(self):
95
99
  impacted_object = ""
@@ -105,7 +109,7 @@ class DbJob:
105
109
  return f"{self.task} for {impacted_object}"
106
110
 
107
111
 
108
- def store_detection_semantics(pic: DbPicture, metadata: Dict[str, Any], store_id: bool):
112
+ def store_detection_semantics(job: DbJob, metadata: Dict[str, Any], store_id: bool):
109
113
  """store the detection returned by the blurring API in the database.
110
114
 
111
115
  The semantics part is stored as annotations, linked to the default account.
@@ -120,13 +124,13 @@ def store_detection_semantics(pic: DbPicture, metadata: Dict[str, Any], store_id
120
124
 
121
125
  tags = metadata.pop("annotations", [])
122
126
 
123
- with db.conn(current_app) as conn, conn.cursor() as cursor:
127
+ with job.reporting_conn.cursor() as cursor:
124
128
  blurring_id = metadata.get("blurring_id")
125
129
  if blurring_id and store_id:
126
130
  # we store the blurring id to be able to unblur the picture later
127
131
  cursor.execute(
128
132
  "UPDATE pictures SET blurring_id = %(blurring_id)s WHERE id = %(id)s",
129
- {"blurring_id": blurring_id, "id": pic.id},
133
+ {"blurring_id": blurring_id, "id": job.pic.id},
130
134
  )
131
135
 
132
136
  if not tags:
@@ -140,21 +144,48 @@ def store_detection_semantics(pic: DbPicture, metadata: Dict[str, Any], store_id
140
144
  # we want to remove all the tags added by the same bluring api previously
141
145
  # it's especially usefull when a picture is blurred multiple times
142
146
  # and if the detection model has been updated between the blurrings
143
- semantics.delete_annotation_tags_from_service(conn, pic.id, service_name="SGBlur", account=default_account_id)
147
+ semantics.delete_annotation_tags_from_service(job.reporting_conn, job.pic.id, service_name="SGBlur", account=default_account_id)
144
148
  try:
145
149
  annotations_to_create = [
146
- annotations.AnnotationCreationParameter(**t, account_id=default_account_id, picture_id=pic.id) for t in tags
150
+ annotations.AnnotationCreationParameter(**t, account_id=default_account_id, picture_id=job.pic.id) for t in tags
147
151
  ]
152
+ for a in annotations_to_create:
153
+ annotations.creation_annotation(a, job.reporting_conn)
148
154
  except Exception as e:
149
155
  # if the detections are not in the correct format, we skip them
150
156
  msg = errors.getMessageFromException(e)
151
- log.error(f"impossible to save blurring detections, skipping it for picture {pic.id}: {msg}")
157
+ if hasattr(e, "payload"):
158
+ msg += f": {e.payload}"
159
+ log.error(f"impossible to save blurring detections, skipping it for picture {job.pic.id}: {msg}")
160
+ job.warning = msg
161
+
162
+
163
+ def update_picture_orientation(conn: psycopg.Connection, db_pic: DbPicture, picturePillow: Image):
164
+ """if the picture is side oriented, we need to check if the blurring API has rotated the picture, and update its size"""
165
+ if db_pic.orientation not in ("6", "8"):
152
166
  return
153
- for a in annotations_to_create:
154
- annotations.creation_annotation(a)
167
+
168
+ new_size = utils.pictures.getPictureSizing(picturePillow)
169
+ if new_size["width"] != db_pic.metadata["width"] or new_size["height"] != db_pic.metadata["height"]:
170
+ with conn.cursor() as cursor:
171
+ # update the new X/Y dimensions and reset the orientation, to tell that it's no longer side oriented
172
+ cursor.execute(
173
+ """UPDATE pictures SET
174
+ exif = exif - 'Exif.Image.Orientation' || jsonb_build_object('Exif.Photo.PixelXDimension', %(width)s, 'Exif.Photo.PixelYDimension', %(height)s),
175
+ metadata = metadata || jsonb_build_object('width', %(width)s, 'height', %(height)s, 'cols', %(cols)s, 'rows', %(rows)s)
176
+ WHERE id = %(id)s
177
+ """,
178
+ {
179
+ "width": new_size["width"],
180
+ "height": new_size["height"],
181
+ "id": db_pic.id,
182
+ "cols": new_size["cols"],
183
+ "rows": new_size["rows"],
184
+ },
185
+ )
155
186
 
156
187
 
157
- def processPictureFiles(pic: DbPicture, config):
188
+ def processPictureFiles(job: DbJob, config):
158
189
  """Generates the files associated with a sequence picture.
159
190
 
160
191
  If needed the image is blurred before the tiles and thumbnail are generated.
@@ -168,6 +199,7 @@ def processPictureFiles(pic: DbPicture, config):
168
199
  config : dict
169
200
  Flask app.config (passed as param to allow using ThreadPoolExecutor)
170
201
  """
202
+ pic = job.pic
171
203
  skipBlur = pic.skip_blurring or config.get("API_BLUR_URL") is None
172
204
  fses = config["FILESYSTEMS"]
173
205
  fs = fses.permanent if skipBlur else fses.tmp
@@ -206,8 +238,12 @@ def processPictureFiles(pic: DbPicture, config):
206
238
  picture = None
207
239
  else:
208
240
  picture = res.image
241
+
242
+ if pic.orientation in ("6", "8"):
243
+ update_picture_orientation(job.reporting_conn, pic, picture)
244
+
209
245
  if res.metadata:
210
- store_detection_semantics(pic, res.metadata, store_id=config["PICTURE_PROCESS_KEEP_UNBLURRED_PARTS"])
246
+ store_detection_semantics(job, res.metadata, store_id=config["PICTURE_PROCESS_KEEP_UNBLURRED_PARTS"])
211
247
 
212
248
  # Delete original unblurred file
213
249
  geovisio.utils.filesystems.removeFsEvenNotFound(fses.tmp, picHdPath)
@@ -287,7 +323,9 @@ class PictureProcessor:
287
323
  if self.app.pool.closed and self.stop:
288
324
  # in some tests, the pool is closed before the worker is stopped, we check this here
289
325
  return
290
- self.check_periodic_tasks()
326
+ if not self.stop:
327
+ # periodic tasks are only checked by permanent workers
328
+ self.check_periodic_tasks()
291
329
  r = process_next_job(self.app)
292
330
  if not r:
293
331
  if self.stop:
@@ -308,7 +346,7 @@ class PictureProcessor:
308
346
  signal.signal(signal.SIGTERM, self._graceful_shutdown)
309
347
 
310
348
  def _graceful_shutdown(self, *args):
311
- log.info("Stoping worker, waiting for last picture processing to finish...")
349
+ log.info("Stopping worker, waiting for last picture processing to finish...")
312
350
  self.stop = True
313
351
 
314
352
  def check_periodic_tasks(self):
@@ -364,16 +402,19 @@ def process_next_job(app):
364
402
  span.set_data("pic_id", job.pic.id)
365
403
  with utils.time.log_elapsed(f"Processing picture {job.pic.id}"):
366
404
  # open another connection for reporting and queries
367
- processPictureFiles(job.pic, app.config)
405
+ processPictureFiles(job, app.config)
368
406
  elif job.task == ProcessTask.delete and job.pic:
369
407
  with sentry_sdk.start_span(description="Deleting picture") as span:
370
408
  span.set_data("pic_id", job.pic.id)
371
409
  with utils.time.log_elapsed(f"Deleting picture {job.pic.id}"):
372
- _delete_picture(job.pic)
410
+ _delete_picture(job.reporting_conn, job.pic)
411
+ elif job.task == ProcessTask.read_metadata and job.pic:
412
+ with utils.time.log_elapsed(f"Reading metadata of picture {job.pic.id}"):
413
+ _read_picture_metadata(job.pic, **(job.args or {}))
373
414
  elif job.task == ProcessTask.dispatch and job.upload_set:
374
415
  with utils.time.log_elapsed(f"Dispatching upload set {job.upload_set.id}"):
375
416
  try:
376
- upload_set.dispatch(job.upload_set.id)
417
+ upload_set.dispatch(job.reporting_conn, job.upload_set.id)
377
418
  except Exception as e:
378
419
  log.exception(f"impossible to dispatch upload set {job.upload_set.id}")
379
420
  raise RecoverableProcessException("Upload set dispatch error: " + errors.getMessageFromException(e)) from e
@@ -399,7 +440,7 @@ def _get_next_job(app):
399
440
  with app.pool.connection() as locking_transaction:
400
441
  with locking_transaction.transaction(), locking_transaction.cursor(row_factory=dict_row) as cursor:
401
442
  r = cursor.execute(
402
- """SELECT j.id, j.picture_id, j.upload_set_id, j.sequence_id, j.task, j.picture_to_delete_id, p.metadata, j.args
443
+ """SELECT j.id, j.picture_id, j.upload_set_id, j.sequence_id, j.task, j.picture_to_delete_id, p.metadata, j.args, p.exif->'Exif.Image.Orientation' as orientation
403
444
  FROM job_queue j
404
445
  LEFT JOIN pictures p ON p.id = j.picture_id
405
446
  ORDER by
@@ -418,7 +459,12 @@ def _get_next_job(app):
418
459
  # (and it will not a foreign key since the picture's row will already have been deleted from the db)
419
460
  pic_id = r["picture_id"] or r["picture_to_delete_id"]
420
461
  db_pic = (
421
- DbPicture(id=pic_id, metadata=r["metadata"], skip_blurring=(r["args"] or {}).get("skip_blurring", False))
462
+ DbPicture(
463
+ id=pic_id,
464
+ metadata=r["metadata"],
465
+ skip_blurring=(r["args"] or {}).get("skip_blurring", False),
466
+ orientation=r["orientation"],
467
+ )
422
468
  if pic_id is not None
423
469
  else None
424
470
  )
@@ -466,7 +512,7 @@ def _get_next_job(app):
466
512
  _finalize_sequence(job)
467
513
  error = e
468
514
 
469
- # we raise an error after the transaction has been comited to be sure to have the state persisted in the database
515
+ # we raise an error after the transaction has been committed to be sure to have the state persisted in the database
470
516
  if error:
471
517
  raise error
472
518
 
@@ -507,9 +553,16 @@ def _finalize_job(conn, job: DbJob):
507
553
  f"The job {job.job_history_id} ({job.label()}) has likely been deleted during the process (it can happen if the picture/upload_set/sequence has been deleted by another process), we don't need to finalize it"
508
554
  )
509
555
  return
556
+
557
+ params = {"id": job.job_history_id}
558
+ fields = [SQL("finished_at = CURRENT_TIMESTAMP")]
559
+ if job.warning:
560
+ fields.append(SQL("warning = %(warn)s"))
561
+ params["warn"] = job.warning
562
+
510
563
  job.reporting_conn.execute(
511
- "UPDATE job_history SET finished_at = CURRENT_TIMESTAMP WHERE id = %(id)s",
512
- {"id": job.job_history_id},
564
+ SQL("UPDATE job_history SET {fields} WHERE id = %(id)s").format(fields=SQL(", ").join(fields)),
565
+ params,
513
566
  )
514
567
  if job.task == ProcessTask.prepare and job.pic:
515
568
  # Note: the status is slowly been deprecated by replacing it with more precise status, and in the end it will be removed
@@ -544,7 +597,7 @@ def _initialize_job(
544
597
  "pic_to_delete": db_pic.id if db_pic and task == ProcessTask.delete else None,
545
598
  "us_id": db_upload_set.id if db_upload_set else None,
546
599
  "task": task.value,
547
- "args": Jsonb(args),
600
+ "args": Jsonb(args) if args else None,
548
601
  },
549
602
  ).fetchone()
550
603
 
@@ -559,6 +612,7 @@ def _initialize_job(
559
612
  upload_set=db_upload_set,
560
613
  task=task,
561
614
  job_history_id=r[0],
615
+ args=args,
562
616
  )
563
617
 
564
618
 
@@ -608,7 +662,58 @@ def _mark_process_as_error(
608
662
  conn.execute("DELETE FROM job_queue WHERE id = %(id)s", {"id": job.job_queue_id})
609
663
 
610
664
 
611
- def _delete_picture(pic: DbPicture):
665
+ def _delete_picture(conn: psycopg.Connection, pic: DbPicture):
612
666
  """Delete a picture from the filesystem"""
613
667
  log.debug(f"Deleting picture files {pic.id}")
668
+
669
+ def check_if_no_workers_preparing():
670
+ try:
671
+ # We try to check if there at some workers preparing this picture, if it's the case, we wait a bit and retry.
672
+ # after some time, if the lock is still not released, we raise a RetryLaterProcessException, to reschedule the whole job later
673
+ conn.execute(
674
+ "SELECT id FROM job_queue WHERE picture_id = %(id)s and task = 'prepare' FOR UPDATE NOWAIT",
675
+ {"id": pic.id},
676
+ )
677
+ return True
678
+ except psycopg.errors.LockNotAvailable:
679
+ logging.debug(f"The picture {pic.id} is being processed, we'll retry later")
680
+ return False
681
+
682
+ _retry_for(check_if_no_workers_preparing, error=f"Picture {pic.id} is being processed")
683
+
684
+ # Delete the row if needed (note that it can have already been deleted (for example if a whole upload_set has been deleted, the `ON DELETE CASCADE` deletes all the pictures's row (but the files still need to be deleted)))
685
+ conn.execute("DELETE FROM pictures WHERE id = %(id)s", {"id": pic.id})
686
+
614
687
  utils.pictures.removeAllFiles(pic.id)
688
+
689
+
690
+ def _retry_for(func, error, timeout=timedelta(minutes=1), sleep=timedelta(seconds=5)):
691
+ import time
692
+
693
+ cur_duration = timedelta(seconds=0)
694
+ while cur_duration < timeout:
695
+ r = func()
696
+ if r:
697
+ return
698
+ cur_duration += sleep
699
+ time.sleep(sleep.total_seconds())
700
+
701
+ raise RetryLaterProcessException(error)
702
+
703
+
704
+ def _read_picture_metadata(picture: DbPicture, read_file=False):
705
+ """Reread the picture metadata.
706
+
707
+ Normally the picture's metadata are read during upload, but sometimes (mainly when the geopic-tag-reader library has been updated),
708
+ we need to read the metadata again.
709
+
710
+ Parameters
711
+ ----------
712
+ picture_id : UUID
713
+ The ID of the picture to read the metadata from
714
+ read_file : bool
715
+ If True, the picture's raw metadata will be read again, else the Exif tools stored in the database will be used (way faster).
716
+ """
717
+
718
+ with db.conn(current_app) as conn:
719
+ utils.pictures.update_picture_metadata(conn, picture.id, read_file)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: geovisio
3
- Version: 2.9.0
3
+ Version: 2.11.0
4
4
  Summary: GeoVisio API - Main
5
5
  Author-email: Adrien PAVIE <panieravide@riseup.net>, Antoine Desbordes <antoine.desbordes@gmail.com>
6
6
  Requires-Python: >=3.10
@@ -11,7 +11,7 @@ Requires-Dist: Flask ~= 3.1
11
11
  Requires-Dist: psycopg[pool] ~= 3.2
12
12
  Requires-Dist: flasgger ~= 0.9.7
13
13
  Requires-Dist: Pillow ~= 11.1
14
- Requires-Dist: Flask-Cors ~= 5.0
14
+ Requires-Dist: Flask-Cors ~= 6.0
15
15
  Requires-Dist: fs ~= 2.4
16
16
  Requires-Dist: fs-s3fs-forked ~= 1.1.4
17
17
  Requires-Dist: flask-compress ~= 1.14
@@ -19,38 +19,39 @@ Requires-Dist: requests ~= 2.31
19
19
  Requires-Dist: yoyo-migrations ~= 9.0
20
20
  Requires-Dist: psycopg-binary ~= 3.2
21
21
  Requires-Dist: python-dotenv ~= 1.1
22
- Requires-Dist: authlib ~= 1.5
22
+ Requires-Dist: authlib ~= 1.6
23
23
  Requires-Dist: Flask-Executor ~= 1.0
24
- Requires-Dist: geopic-tag-reader[write-exif] == 1.4.2
24
+ Requires-Dist: geopic-tag-reader[write-exif] ~= 1.8.0
25
25
  Requires-Dist: rfeed ~= 1.1.1
26
26
  Requires-Dist: sentry-sdk[flask] ~= 2.24
27
27
  Requires-Dist: pygeofilter[backend-native] ~= 0.3.1
28
28
  Requires-Dist: python-dateutil ~= 2.9.0
29
29
  Requires-Dist: tzdata ~= 2025.2
30
30
  Requires-Dist: croniter ~= 6.0.0
31
- Requires-Dist: pydantic ~= 2.11
31
+ Requires-Dist: pydantic ~= 2.12
32
32
  Requires-Dist: pydantic-extra-types ~= 2.7
33
33
  Requires-Dist: flask-babel ~= 4.0.0
34
- Requires-Dist: geojson-pydantic ~= 1.2.0
34
+ Requires-Dist: geojson-pydantic ~= 2.0.0
35
35
  Requires-Dist: email-validator ~= 2.2.0
36
36
  Requires-Dist: multipart>=1.2.1
37
+ Requires-Dist: gunicorn ~= 23.0.0
37
38
  Requires-Dist: flit ~= 3.9.0 ; extra == "build"
38
- Requires-Dist: coverage ~= 7.7 ; extra == "dev"
39
- Requires-Dist: protobuf ~= 4.21 ; extra == "dev"
40
- Requires-Dist: mapbox-vector-tile ~= 2.0 ; extra == "dev"
39
+ Requires-Dist: coverage ~= 7.9 ; extra == "dev"
40
+ Requires-Dist: protobuf ~= 5.26 ; extra == "dev"
41
+ Requires-Dist: mapbox-vector-tile ~= 2.1 ; extra == "dev"
41
42
  Requires-Dist: pystac ~= 1.9 ; extra == "dev"
42
- Requires-Dist: pytest ~= 8.3 ; extra == "dev"
43
- Requires-Dist: pytest-datafiles ~= 2.0 ; extra == "dev"
43
+ Requires-Dist: pytest ~= 8.4 ; extra == "dev"
44
+ Requires-Dist: pytest-datafiles ~= 3.0 ; extra == "dev"
44
45
  Requires-Dist: pyexiv2 ~= 2.15 ; extra == "dev"
45
- Requires-Dist: testcontainers ~= 4.1 ; extra == "dev"
46
+ Requires-Dist: testcontainers ~= 4.10 ; extra == "dev"
46
47
  Requires-Dist: requests-mock ~= 1.11 ; extra == "dev"
47
48
  Requires-Dist: black ~= 25.1 ; extra == "dev"
48
49
  Requires-Dist: pre-commit ~= 4.2 ; extra == "dev"
49
50
  Requires-Dist: pyyaml ~= 6.0 ; extra == "dev"
50
51
  Requires-Dist: openapi-spec-validator ~= 0.7 ; extra == "dev"
51
52
  Requires-Dist: stac-api-validator ~= 0.6.4 ; extra == "dev"
52
- Requires-Dist: mkdocs-material ~= 9.6.9 ; extra == "docs"
53
- Requires-Dist: mkdocs-swagger-ui-tag ~= 0.6.11 ; extra == "docs"
53
+ Requires-Dist: mkdocs-material ~= 9.6.14 ; extra == "docs"
54
+ Requires-Dist: mkdocs-swagger-ui-tag ~= 0.7.1 ; extra == "docs"
54
55
  Project-URL: Home, https://gitlab.com/panoramax/server/api
55
56
  Project-URL: Source Code, https://gitlab.com/panoramax/server/api
56
57
  Provides-Extra: build