geovisio 2.7.1__py3-none-any.whl → 2.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. geovisio/__init__.py +10 -2
  2. geovisio/admin_cli/__init__.py +3 -1
  3. geovisio/admin_cli/user.py +75 -0
  4. geovisio/config_app.py +87 -4
  5. geovisio/templates/main.html +2 -2
  6. geovisio/templates/viewer.html +3 -3
  7. geovisio/translations/da/LC_MESSAGES/messages.mo +0 -0
  8. geovisio/translations/da/LC_MESSAGES/messages.po +850 -0
  9. geovisio/translations/de/LC_MESSAGES/messages.mo +0 -0
  10. geovisio/translations/de/LC_MESSAGES/messages.po +97 -1
  11. geovisio/translations/el/LC_MESSAGES/messages.mo +0 -0
  12. geovisio/translations/en/LC_MESSAGES/messages.mo +0 -0
  13. geovisio/translations/en/LC_MESSAGES/messages.po +210 -127
  14. geovisio/translations/eo/LC_MESSAGES/messages.mo +0 -0
  15. geovisio/translations/eo/LC_MESSAGES/messages.po +790 -0
  16. geovisio/translations/es/LC_MESSAGES/messages.mo +0 -0
  17. geovisio/translations/fi/LC_MESSAGES/messages.mo +0 -0
  18. geovisio/translations/fr/LC_MESSAGES/messages.mo +0 -0
  19. geovisio/translations/fr/LC_MESSAGES/messages.po +39 -2
  20. geovisio/translations/hu/LC_MESSAGES/messages.mo +0 -0
  21. geovisio/translations/it/LC_MESSAGES/messages.mo +0 -0
  22. geovisio/translations/it/LC_MESSAGES/messages.po +875 -0
  23. geovisio/translations/ja/LC_MESSAGES/messages.mo +0 -0
  24. geovisio/translations/ja/LC_MESSAGES/messages.po +719 -0
  25. geovisio/translations/ko/LC_MESSAGES/messages.mo +0 -0
  26. geovisio/translations/messages.pot +191 -122
  27. geovisio/translations/nl/LC_MESSAGES/messages.mo +0 -0
  28. geovisio/translations/pl/LC_MESSAGES/messages.mo +0 -0
  29. geovisio/translations/pl/LC_MESSAGES/messages.po +727 -0
  30. geovisio/translations/zh_Hant/LC_MESSAGES/messages.mo +0 -0
  31. geovisio/translations/zh_Hant/LC_MESSAGES/messages.po +719 -0
  32. geovisio/utils/auth.py +80 -8
  33. geovisio/utils/link.py +3 -2
  34. geovisio/utils/model_query.py +55 -0
  35. geovisio/utils/pictures.py +12 -43
  36. geovisio/utils/semantics.py +120 -0
  37. geovisio/utils/sequences.py +10 -1
  38. geovisio/utils/tokens.py +5 -3
  39. geovisio/utils/upload_set.py +50 -15
  40. geovisio/utils/website.py +50 -0
  41. geovisio/web/annotations.py +17 -0
  42. geovisio/web/auth.py +9 -5
  43. geovisio/web/collections.py +217 -61
  44. geovisio/web/configuration.py +17 -1
  45. geovisio/web/docs.py +64 -53
  46. geovisio/web/items.py +220 -96
  47. geovisio/web/map.py +48 -18
  48. geovisio/web/pages.py +240 -0
  49. geovisio/web/params.py +17 -0
  50. geovisio/web/prepare.py +165 -0
  51. geovisio/web/stac.py +17 -4
  52. geovisio/web/tokens.py +14 -4
  53. geovisio/web/upload_set.py +10 -4
  54. geovisio/web/users.py +176 -44
  55. geovisio/workers/runner_pictures.py +61 -22
  56. {geovisio-2.7.1.dist-info → geovisio-2.8.0.dist-info}/METADATA +5 -4
  57. geovisio-2.8.0.dist-info/RECORD +89 -0
  58. geovisio-2.7.1.dist-info/RECORD +0 -70
  59. {geovisio-2.7.1.dist-info → geovisio-2.8.0.dist-info}/LICENSE +0 -0
  60. {geovisio-2.7.1.dist-info → geovisio-2.8.0.dist-info}/WHEEL +0 -0
geovisio/web/users.py CHANGED
@@ -1,43 +1,64 @@
1
+ from typing import List, Optional
2
+ from uuid import UUID
1
3
  import flask
2
- from flask import request, current_app, url_for
4
+ from flask import redirect, request, current_app, session, url_for
3
5
  from flask_babel import gettext as _
6
+ from pydantic import BaseModel, Field, ValidationError, computed_field
4
7
  from geovisio.utils import auth, db
5
8
  from geovisio import errors
6
- from psycopg.rows import dict_row
9
+ from psycopg.rows import dict_row, class_row
7
10
  from psycopg.sql import SQL
8
11
 
12
+ from geovisio.utils.link import Link, make_link
13
+ from geovisio.utils.model_query import get_db_params_and_values
14
+ from geovisio.utils.params import validation_error
9
15
  from geovisio.web import stac
16
+ from geovisio.web.auth import NEXT_URL_KEY
10
17
  from geovisio.web.utils import get_root_link
11
18
 
12
19
  bp = flask.Blueprint("user", __name__, url_prefix="/api/users")
13
20
 
14
21
 
15
- def _get_user_info(id, name):
16
- userMapUrl = (
17
- flask.url_for("map.getUserTile", userId=id, x="11111111", y="22222222", z="33333333", format="mvt", _external=True)
18
- .replace("11111111", "{x}")
19
- .replace("22222222", "{y}")
20
- .replace("33333333", "{z}")
21
- )
22
- response = {
23
- "id": id,
24
- "name": name,
25
- "links": [
26
- {"rel": "catalog", "type": "application/json", "href": flask.url_for("stac.getUserCatalog", userId=id, _external=True)},
27
- {
28
- "rel": "collection",
29
- "type": "application/json",
30
- "href": flask.url_for("stac_collections.getUserCollection", userId=id, _external=True),
31
- },
32
- {
33
- "rel": "user-xyz",
34
- "type": "application/vnd.mapbox-vector-tile",
35
- "href": userMapUrl,
36
- "title": "Pictures and sequences vector tiles for a given user",
37
- },
38
- ],
39
- }
40
- return flask.jsonify(response)
22
+ class UserInfo(BaseModel):
23
+ name: str
24
+ """Name of the user"""
25
+ id: UUID
26
+ """Unique identifier of the user"""
27
+ collaborative_metadata: Optional[bool] = None
28
+ """If true, the user can edit the metadata of all sequences. If unset, default to the instance's default configuration."""
29
+
30
+ tos_accepted: Optional[bool] = None
31
+ """True means the user has accepted the terms of service (tos). Can only be seen by the user itself"""
32
+
33
+ @computed_field
34
+ @property
35
+ def links(self) -> List[Link]:
36
+ userMapUrl = (
37
+ flask.url_for("map.getUserTile", userId=self.id, x="11111111", y="22222222", z="33333333", format="mvt", _external=True)
38
+ .replace("11111111", "{x}")
39
+ .replace("22222222", "{y}")
40
+ .replace("33333333", "{z}")
41
+ )
42
+ return [
43
+ make_link(rel="catalog", route="stac.getUserCatalog", userId=self.id),
44
+ make_link(rel="collection", route="stac_collections.getUserCollection", userId=self.id),
45
+ Link(
46
+ rel="user-xyz",
47
+ type="application/vnd.mapbox-vector-tile",
48
+ title="Pictures and sequences vector tiles for a given user",
49
+ href=userMapUrl,
50
+ ),
51
+ ]
52
+
53
+
54
+ def _get_user_info(account: auth.Account):
55
+ user_info = UserInfo(id=account.id, name=account.name, collaborative_metadata=account.collaborative_metadata)
56
+ logged_account = auth.get_current_account()
57
+ if logged_account is not None and account.id == logged_account.id:
58
+ # we show the term of service acceptance only if the user is the logged user
59
+ user_info.tos_accepted = account.tos_accepted
60
+
61
+ return user_info.model_dump(exclude_unset=True), 200, {"Content-Type": "application/json"}
41
62
 
42
63
 
43
64
  @bp.route("/me")
@@ -55,7 +76,7 @@ def getMyUserInfo(account):
55
76
  schema:
56
77
  $ref: '#/components/schemas/GeoVisioUser'
57
78
  """
58
- return _get_user_info(account.id, account.name)
79
+ return _get_user_info(account)
59
80
 
60
81
 
61
82
  @bp.route("/<uuid:userId>")
@@ -79,21 +100,29 @@ def getUserInfo(userId):
79
100
  schema:
80
101
  $ref: '#/components/schemas/GeoVisioUser'
81
102
  """
82
- account = db.fetchone(current_app, SQL("SELECT name, id FROM accounts WHERE id = %s"), [userId], row_factory=dict_row)
103
+ account = db.fetchone(
104
+ current_app,
105
+ SQL("SELECT name, id::text, collaborative_metadata, role, tos_accepted FROM accounts WHERE id = %s"),
106
+ [userId],
107
+ row_factory=class_row(auth.Account),
108
+ )
83
109
  if not account:
84
110
  raise errors.InvalidAPIUsage(_("Impossible to find user"), status_code=404)
85
111
 
86
- return _get_user_info(account["id"], account["name"])
112
+ return _get_user_info(account)
87
113
 
88
114
 
89
115
  @bp.route("/me/catalog")
90
116
  @auth.login_required_with_redirect()
91
117
  def getMyCatalog(account):
92
- """Get current logged user catalog
118
+ """Get current logged user catalog.
119
+
120
+ Note that this route is deprecated in favor of `/api/users/me/collection`. This new route provides more information and offers more filtering and sorting options.
93
121
  ---
94
122
  tags:
95
123
  - Users
96
124
  - Sequences
125
+ deprecated: true
97
126
  responses:
98
127
  200:
99
128
  description: the Catalog listing all sequences associated to given user. Note that it's similar to the user's colletion, but with less metadata since a STAC collection is an enhanced STAC catalog.
@@ -102,18 +131,37 @@ def getMyCatalog(account):
102
131
  schema:
103
132
  $ref: '#/components/schemas/GeoVisioCatalog'
104
133
  """
105
- return flask.redirect(flask.url_for("stac.getUserCatalog", userId=account.id, _external=True))
134
+ return flask.redirect(
135
+ flask.url_for(
136
+ "stac.getUserCatalog",
137
+ userId=account.id,
138
+ limit=request.args.get("limit"),
139
+ page=request.args.get("page"),
140
+ _external=True,
141
+ )
142
+ )
106
143
 
107
144
 
108
145
  @bp.route("/me/collection")
109
146
  @auth.login_required_with_redirect()
110
147
  def getMyCollection(account):
111
148
  """Get current logged user collection
149
+
150
+ Note that the result can also be a CSV file, if the "Accept" header is set to "text/csv", or if the "format" query parameter is set to "csv".
151
+
112
152
  ---
113
153
  tags:
114
154
  - Users
115
155
  - Sequences
116
156
  parameters:
157
+ - name: format
158
+ in: query
159
+ description: Expected output format (STAC JSON or a csv file)
160
+ required: false
161
+ schema:
162
+ type: string
163
+ enum: [csv, json]
164
+ default: json
117
165
  - $ref: '#/components/parameters/STAC_collections_limit'
118
166
  - $ref: '#/components/parameters/STAC_collections_filter'
119
167
  - $ref: '#/components/parameters/STAC_bbox'
@@ -125,19 +173,14 @@ def getMyCollection(account):
125
173
  application/json:
126
174
  schema:
127
175
  $ref: '#/components/schemas/GeoVisioCollectionOfCollection'
176
+
177
+ text/csv:
178
+ schema:
179
+ $ref: '#/components/schemas/GeoVisioCSVCollections'
128
180
  """
181
+ from geovisio.web.collections import getUserCollection
129
182
 
130
- return flask.redirect(
131
- flask.url_for(
132
- "stac_collections.getUserCollection",
133
- userId=account.id,
134
- filter=request.args.get("filter"),
135
- limit=request.args.get("limit"),
136
- sortby=request.args.get("sortby"),
137
- bbox=request.args.get("bbox"),
138
- _external=True,
139
- )
140
- )
183
+ return getUserCollection(userId=account.id, userIdMatchesAccount=True)
141
184
 
142
185
 
143
186
  @bp.route("/search")
@@ -272,3 +315,92 @@ LIMIT {limit};"""
272
315
  if r["has_seq"]
273
316
  ],
274
317
  }
318
+
319
+
320
+ class UserConfiguration(BaseModel):
321
+ collaborative_metadata: Optional[bool] = None
322
+ """If true, all sequences's metadata will be, by default, editable by all users.
323
+
324
+ If not set, it will default to the instance default collaborative editing policy."""
325
+
326
+ def has_override(self) -> bool:
327
+ return bool(self.model_fields_set)
328
+
329
+
330
+ @bp.route("/me", methods=["PATCH"])
331
+ @auth.login_required()
332
+ def patchUserConfiguration(account):
333
+ """Edit the current user configuration
334
+
335
+ ---
336
+ tags:
337
+ - Users
338
+ requestBody:
339
+ content:
340
+ application/json:
341
+ schema:
342
+ $ref: '#/components/schemas/GeoVisioUserConfiguration'
343
+ security:
344
+ - bearerToken: []
345
+ - cookieAuth: []
346
+ responses:
347
+ 200:
348
+ description: the user configuration
349
+ content:
350
+ application/json:
351
+ schema:
352
+ $ref: '#/components/schemas/GeoVisioUser'
353
+ """
354
+ metadata = None
355
+ try:
356
+ if request.is_json and request.json:
357
+ metadata = UserConfiguration(**request.json)
358
+ except ValidationError as ve:
359
+ raise errors.InvalidAPIUsage(_("Impossible to parse parameters"), payload=validation_error(ve))
360
+
361
+ if not metadata:
362
+ return _get_user_info(account)
363
+ params = get_db_params_and_values(metadata)
364
+ if metadata.has_override():
365
+
366
+ fields = params.fields_for_set_list()
367
+
368
+ account = db.fetchone(
369
+ current_app,
370
+ SQL("UPDATE accounts SET {fields} WHERE id = %(account_id)s RETURNING *").format(fields=SQL(", ").join(fields)),
371
+ params.params_as_dict | {"account_id": account.id},
372
+ row_factory=class_row(auth.Account),
373
+ )
374
+
375
+ return _get_user_info(account)
376
+
377
+
378
+ @bp.route("/me/accept_tos", methods=["POST"])
379
+ @auth.login_required()
380
+ def accept_tos(account: auth.Account):
381
+ """
382
+ Accept the terms of service for the current user
383
+ ---
384
+ tags:
385
+ - Auth
386
+ responses:
387
+ 200:
388
+ description: the user configuration
389
+ content:
390
+ application/json:
391
+ schema:
392
+ $ref: '#/components/schemas/GeoVisioUser'
393
+ """
394
+ # Note: accepting twice does not change the accepted_at date
395
+ account = db.fetchone(
396
+ current_app,
397
+ SQL("UPDATE accounts SET tos_accepted_at = COALESCE(tos_accepted_at, NOW()) WHERE id = %(account_id)s RETURNING *"),
398
+ {"account_id": account.id},
399
+ row_factory=class_row(auth.Account),
400
+ )
401
+
402
+ # we persist in the cookie the fact that the tos have been accepted
403
+ session[auth.ACCOUNT_KEY] = account.model_dump(exclude_none=True)
404
+ session.permanent = True
405
+
406
+ return _get_user_info(account)
@@ -6,13 +6,14 @@ from geovisio.utils import db, sequences, upload_set
6
6
  import psycopg
7
7
  from psycopg.rows import dict_row
8
8
  from psycopg.sql import SQL
9
+ from psycopg.types.json import Jsonb
9
10
  import sentry_sdk
10
11
  from geovisio import errors
11
12
  from dataclasses import dataclass
12
13
  import logging
13
14
  from contextlib import contextmanager
14
15
  from enum import Enum
15
- from typing import Any, Optional
16
+ from typing import Any, Dict, Optional
16
17
  import threading
17
18
  from uuid import UUID
18
19
  from croniter import croniter
@@ -21,7 +22,7 @@ import geovisio.utils.filesystems
21
22
 
22
23
  log = logging.getLogger("geovisio.runner_pictures")
23
24
 
24
- PICTURE_PROCESS_MAX_RETRY = 10 # Number of times a job will be retryed if there is a `RecoverableProcessException` during process (like if the blurring api is not reachable).
25
+ PROCESS_MAX_RETRY = 5 # Number of times a job will be retryed if there is a `RecoverableProcessException` during process (like if the blurring api is not reachable).
25
26
 
26
27
 
27
28
  class PictureBackgroundProcessor(object):
@@ -50,9 +51,6 @@ class PictureBackgroundProcessor(object):
50
51
  return self.executor.submit(worker.process_jobs)
51
52
 
52
53
 
53
- # background_processor = PictureBackgroundProcessor()
54
-
55
-
56
54
  class ProcessTask(str, Enum):
57
55
  prepare = "prepare"
58
56
  delete = "delete"
@@ -64,6 +62,7 @@ class ProcessTask(str, Enum):
64
62
  class DbPicture:
65
63
  id: UUID
66
64
  metadata: dict
65
+ skip_blurring: bool
67
66
 
68
67
  def blurred_by_author(self):
69
68
  return self.metadata.get("blurredByAuthor", False)
@@ -118,7 +117,7 @@ def processPictureFiles(pic: DbPicture, config):
118
117
  config : dict
119
118
  Flask app.config (passed as param to allow using ThreadPoolExecutor)
120
119
  """
121
- skipBlur = pic.blurred_by_author() or config.get("API_BLUR_URL") is None
120
+ skipBlur = pic.skip_blurring or config.get("API_BLUR_URL") is None
122
121
  fses = config["FILESYSTEMS"]
123
122
  fs = fses.permanent if skipBlur else fses.tmp
124
123
  picHdPath = utils.pictures.getHDPicturePath(pic.id)
@@ -132,8 +131,6 @@ def processPictureFiles(pic: DbPicture, config):
132
131
  raise Exception(f"Impossible to find picture file: {picHdPath}")
133
132
 
134
133
  with fs.openbin(picHdPath) as pictureBytes:
135
- picture = Image.open(pictureBytes)
136
-
137
134
  # Create picture folders for this specific picture
138
135
  picDerivatesFolder = utils.pictures.getPictureFolderPath(pic.id)
139
136
  fses.derivates.makedirs(picDerivatesFolder, recreate=True)
@@ -143,9 +140,14 @@ def processPictureFiles(pic: DbPicture, config):
143
140
  if not skipBlur:
144
141
  with sentry_sdk.start_span(description="Blurring picture"):
145
142
  try:
146
- picture = utils.pictures.createBlurredHDPicture(fses.permanent, config.get("API_BLUR_URL"), pictureBytes, picHdPath)
143
+ picture = utils.pictures.createBlurredHDPicture(
144
+ fses.permanent,
145
+ config.get("API_BLUR_URL"),
146
+ pictureBytes,
147
+ picHdPath,
148
+ )
147
149
  except Exception as e:
148
- logging.exception(e)
150
+ log.exception(f"impossible to blur picture {pic.id}")
149
151
  raise RecoverableProcessException("Blur API failure: " + errors.getMessageFromException(e)) from e
150
152
 
151
153
  # Delete original unblurred file
@@ -166,15 +168,26 @@ def processPictureFiles(pic: DbPicture, config):
166
168
  else:
167
169
  # Make sure image rotation is always applied
168
170
  # -> Not necessary on pictures from blur API, as SGBlur ensures rotation is always applied
171
+ picture = Image.open(pictureBytes)
169
172
  picture = ImageOps.exif_transpose(picture)
170
173
 
171
174
  # Always pre-generate thumbnail
172
- utils.pictures.createThumbPicture(fses.derivates, picture, picDerivatesFolder + "/thumb.jpg", pic.metadata["type"])
175
+ utils.pictures.createThumbPicture(
176
+ fses.derivates,
177
+ picture,
178
+ picDerivatesFolder + "/thumb.jpg",
179
+ pic.metadata["type"],
180
+ )
173
181
 
174
182
  # Create SD and tiles
175
183
  if config.get("PICTURE_PROCESS_DERIVATES_STRATEGY") == "PREPROCESS":
176
184
  utils.pictures.generatePictureDerivates(
177
- fses.derivates, picture, pic.metadata, picDerivatesFolder, pic.metadata["type"], skipThumbnail=True
185
+ fses.derivates,
186
+ picture,
187
+ pic.metadata,
188
+ picDerivatesFolder,
189
+ pic.metadata["type"],
190
+ skipThumbnail=True,
178
191
  )
179
192
 
180
193
 
@@ -300,7 +313,11 @@ def process_next_job(app):
300
313
  _delete_picture(job.pic)
301
314
  elif job.task == ProcessTask.dispatch and job.upload_set:
302
315
  with utils.time.log_elapsed(f"Dispatching upload set {job.upload_set.id}"):
303
- upload_set.dispatch(job.upload_set.id)
316
+ try:
317
+ upload_set.dispatch(job.upload_set.id)
318
+ except Exception as e:
319
+ log.exception(f"impossible to dispatch upload set {job.upload_set.id}")
320
+ raise RecoverableProcessException("Upload set dispatch error: " + errors.getMessageFromException(e)) from e
304
321
  elif job.task == ProcessTask.finalize and job.seq:
305
322
  with utils.time.log_elapsed(f"Finalizing sequence {job.seq.id}"):
306
323
  with job.reporting_conn.cursor(row_factory=dict_row) as cursor:
@@ -323,7 +340,7 @@ def _get_next_job(app):
323
340
  with app.pool.connection() as locking_transaction:
324
341
  with locking_transaction.transaction(), locking_transaction.cursor(row_factory=dict_row) as cursor:
325
342
  r = cursor.execute(
326
- """SELECT j.id, j.picture_id, j.upload_set_id, j.sequence_id, j.task, j.picture_to_delete_id, p.metadata
343
+ """SELECT j.id, j.picture_id, j.upload_set_id, j.sequence_id, j.task, j.picture_to_delete_id, p.metadata, j.args
327
344
  FROM job_queue j
328
345
  LEFT JOIN pictures p ON p.id = j.picture_id
329
346
  ORDER by
@@ -341,7 +358,11 @@ def _get_next_job(app):
341
358
  # picture id can either be in `picture_id` (and it will be a foreign key to picture) or in `picture_to_delete_id`
342
359
  # (and it will not a foreign key since the picture's row will already have been deleted from the db)
343
360
  pic_id = r["picture_id"] or r["picture_to_delete_id"]
344
- db_pic = DbPicture(id=pic_id, metadata=r["metadata"]) if pic_id is not None else None
361
+ db_pic = (
362
+ DbPicture(id=pic_id, metadata=r["metadata"], skip_blurring=(r["args"] or {}).get("skip_blurring", False))
363
+ if pic_id is not None
364
+ else None
365
+ )
345
366
  db_seq = DbSequence(id=r["sequence_id"]) if r["sequence_id"] is not None else None
346
367
  db_upload_set = DbUploadSet(id=r["upload_set_id"]) if r["upload_set_id"] is not None else None
347
368
 
@@ -353,6 +374,7 @@ def _get_next_job(app):
353
374
  db_seq=db_seq,
354
375
  db_upload_set=db_upload_set,
355
376
  task=ProcessTask(r["task"]),
377
+ args=r["args"],
356
378
  )
357
379
  try:
358
380
  yield job
@@ -363,7 +385,13 @@ def _get_next_job(app):
363
385
  except RecoverableProcessException as e:
364
386
  _mark_process_as_error(locking_transaction, job, e, recoverable=True)
365
387
  except RetryLaterProcessException as e:
366
- _mark_process_as_error(locking_transaction, job, e, recoverable=True, mark_as_error=False)
388
+ _mark_process_as_error(
389
+ locking_transaction,
390
+ job,
391
+ e,
392
+ recoverable=True,
393
+ mark_as_error=False,
394
+ )
367
395
  except InterruptedError as interruption:
368
396
  log.error(f"Interruption received, stoping job {job.label()}")
369
397
  # starts a new connection, since the current one can be corrupted by the exception
@@ -410,7 +438,10 @@ def _finalize_job(conn, job: DbJob):
410
438
  try:
411
439
  # we try to see if our job_history row is still here.
412
440
  # It can have been removed if the object this job was preparing has been deleted during the process (since the job_history table store foreign keys)
413
- job.reporting_conn.execute("SELECT id FROM job_history WHERE id = %(id)s FOR UPDATE NOWAIT", {"id": job.job_history_id})
441
+ job.reporting_conn.execute(
442
+ "SELECT id FROM job_history WHERE id = %(id)s FOR UPDATE NOWAIT",
443
+ {"id": job.job_history_id},
444
+ )
414
445
  except psycopg.errors.LockNotAvailable:
415
446
  logging.info(
416
447
  f"The job {job.job_history_id} ({job.label()}) has likely been deleted during the process (it can happen if the picture/upload_set/sequence has been deleted by another process), we don't need to finalize it"
@@ -423,7 +454,7 @@ def _finalize_job(conn, job: DbJob):
423
454
  if job.task == ProcessTask.prepare and job.pic:
424
455
  # Note: the status is slowly been deprecated by replacing it with more precise status, and in the end it will be removed
425
456
  job.reporting_conn.execute(
426
- "UPDATE pictures SET status = 'ready', preparing_status = 'prepared' WHERE id = %(pic_id)s",
457
+ "UPDATE pictures SET status = (CASE WHEN status = 'hidden' THEN 'hidden' ELSE 'ready' END)::picture_status, preparing_status = 'prepared' WHERE id = %(pic_id)s",
427
458
  {"pic_id": job.pic.id},
428
459
  )
429
460
 
@@ -440,10 +471,11 @@ def _initialize_job(
440
471
  db_seq: Optional[DbSequence],
441
472
  db_upload_set: Optional[DbUploadSet],
442
473
  task: ProcessTask,
474
+ args: Optional[Dict[Any, Any]],
443
475
  ) -> DbJob:
444
476
  r = reporting_conn.execute(
445
- """INSERT INTO job_history(job_id, picture_id, sequence_id, upload_set_id, picture_to_delete_id, job_task)
446
- VALUES (%(job_id)s, %(pic_id)s, %(seq_id)s, %(us_id)s, %(pic_to_delete)s, %(task)s)
477
+ """INSERT INTO job_history(job_id, picture_id, sequence_id, upload_set_id, picture_to_delete_id, job_task, args)
478
+ VALUES (%(job_id)s, %(pic_id)s, %(seq_id)s, %(us_id)s, %(pic_to_delete)s, %(task)s, %(args)s)
447
479
  RETURNING id""",
448
480
  {
449
481
  "job_id": job_queue_id,
@@ -452,6 +484,7 @@ def _initialize_job(
452
484
  "pic_to_delete": db_pic.id if db_pic and task == ProcessTask.delete else None,
453
485
  "us_id": db_upload_set.id if db_upload_set else None,
454
486
  "task": task.value,
487
+ "args": Jsonb(args),
455
488
  },
456
489
  ).fetchone()
457
490
 
@@ -469,7 +502,13 @@ def _initialize_job(
469
502
  )
470
503
 
471
504
 
472
- def _mark_process_as_error(conn, job: DbJob, e: Exception, recoverable: bool = False, mark_as_error: bool = True):
505
+ def _mark_process_as_error(
506
+ conn,
507
+ job: DbJob,
508
+ e: Exception,
509
+ recoverable: bool = False,
510
+ mark_as_error: bool = True,
511
+ ):
473
512
  job.reporting_conn.execute(
474
513
  """UPDATE job_history SET
475
514
  error = %(err)s, finished_at = CURRENT_TIMESTAMP
@@ -485,7 +524,7 @@ def _mark_process_as_error(conn, job: DbJob, e: Exception, recoverable: bool = F
485
524
  RETURNING nb_errors""",
486
525
  {"err": str(e), "id": job.job_queue_id},
487
526
  ).fetchone()
488
- if nb_error and nb_error[0] > PICTURE_PROCESS_MAX_RETRY:
527
+ if nb_error and nb_error[0] > PROCESS_MAX_RETRY:
489
528
  logging.info(f"Job {job.label()} has failed {nb_error} times, we stop trying to process it.")
490
529
  recoverable = False
491
530
  else:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: geovisio
3
- Version: 2.7.1
3
+ Version: 2.8.0
4
4
  Summary: GeoVisio API - Main
5
5
  Author-email: Adrien PAVIE <panieravide@riseup.net>, Antoine Desbordes <antoine.desbordes@gmail.com>
6
6
  Requires-Python: >=3.10
@@ -20,17 +20,18 @@ Requires-Dist: psycopg-binary ~= 3.1
20
20
  Requires-Dist: python-dotenv ~= 0.21
21
21
  Requires-Dist: authlib ~= 1.2
22
22
  Requires-Dist: Flask-Executor ~= 1.0
23
- Requires-Dist: geopic-tag-reader[write-exif] == 1.3.1
23
+ Requires-Dist: geopic-tag-reader[write-exif] == 1.4.2
24
24
  Requires-Dist: rfeed ~= 1.1.1
25
25
  Requires-Dist: sentry-sdk[flask] ~= 1.31
26
26
  Requires-Dist: pygeofilter[backend-native] ~= 0.2.4
27
- Requires-Dist: python-dateutil ~= 2.8.2
27
+ Requires-Dist: python-dateutil ~= 2.9.0
28
28
  Requires-Dist: tzdata ~= 2024.1
29
29
  Requires-Dist: croniter ~= 2.0.5
30
30
  Requires-Dist: pydantic ~= 2.7
31
31
  Requires-Dist: pydantic-extra-types ~= 2.7
32
32
  Requires-Dist: flask-babel ~= 4.0.0
33
33
  Requires-Dist: geojson-pydantic ~= 1.1.0
34
+ Requires-Dist: email-validator ~= 2.2.0
34
35
  Requires-Dist: flit ~= 3.9.0 ; extra == "build"
35
36
  Requires-Dist: coverage ~= 6.5 ; extra == "dev"
36
37
  Requires-Dist: protobuf ~= 4.21 ; extra == "dev"
@@ -45,7 +46,7 @@ Requires-Dist: black ~= 24.1 ; extra == "dev"
45
46
  Requires-Dist: pre-commit ~= 3.3 ; extra == "dev"
46
47
  Requires-Dist: pyyaml ~= 6.0 ; extra == "dev"
47
48
  Requires-Dist: openapi-spec-validator ~= 0.7 ; extra == "dev"
48
- Requires-Dist: stac-api-validator ~= 0.6.3 ; extra == "dev"
49
+ Requires-Dist: stac-api-validator ~= 0.6.4 ; extra == "dev"
49
50
  Requires-Dist: mkdocs-material ~= 9.5.21 ; extra == "docs"
50
51
  Requires-Dist: mkdocs-swagger-ui-tag ~= 0.6.10 ; extra == "docs"
51
52
  Project-URL: Home, https://gitlab.com/panoramax/server/api
@@ -0,0 +1,89 @@
1
+ geovisio/__init__.py,sha256=SPh7-7e0TuxuQNMcF9hDwkg7hW0bJDpbcRnxwu_5DQk,7145
2
+ geovisio/config_app.py,sha256=8RlBxn2LcA71o0jcfR5UrVz12ig985MNUgCj54ZydpA,14356
3
+ geovisio/db_migrations.py,sha256=9lHkyG_RiCWzrFkfwhkslScUsbCZScN-KVhkXrtnPDo,4560
4
+ geovisio/errors.py,sha256=uTn-kI7SUl5OPB8Mv3Qqu7Ucp5JvcqWPQFfgLCqsEpI,1376
5
+ geovisio/admin_cli/__init__.py,sha256=1e0hX771-3iG8eBcNmVvUYyg8qXnpng-9YWvi3MI3Kg,3248
6
+ geovisio/admin_cli/cleanup.py,sha256=G85I7rrfPJwaArL6MQAnC04Ye9wWciA-Yqu5iv23uJ0,4862
7
+ geovisio/admin_cli/db.py,sha256=mJ-cGuOAAsg-ovbP9L1kyL4xE0C4bYRuozzqQkaFyw8,897
8
+ geovisio/admin_cli/default_account_tokens.py,sha256=W-v5uPjCBvAujjAUx1HrfgjPj-tEyncb-EUMLpsWc9w,469
9
+ geovisio/admin_cli/reorder_sequences.py,sha256=LKKzdO2w4N-cQmi6rqKHKYG5YGzPxYRTbnfcPKakuYM,2826
10
+ geovisio/admin_cli/sequence_heading.py,sha256=BEPuRfCDXXpqSSzK2ysrxHf0OD4THzrMI_YK2uXQlGk,633
11
+ geovisio/admin_cli/user.py,sha256=4ml2E_aphz3I3NcuUPB2dwe_jXhcE7AGa0R5VTm3_ik,2753
12
+ geovisio/templates/main.html,sha256=VDVQwCZ1mNjH7sH4VOIdn8gM09R9LJZX49SPtA2VEzM,2963
13
+ geovisio/templates/viewer.html,sha256=JErXdU2ujj4LdMHgQbYNCTfKuYGEXbJTQwBE-K_MNXQ,892
14
+ geovisio/translations/messages.pot,sha256=SUPAgov3RzwVw0LNOMn2NkTthXfirbBDIQK_2BaZ2FI,19445
15
+ geovisio/translations/da/LC_MESSAGES/messages.mo,sha256=zFlDCgA4l-1MamoKMXfFR3diQk67gXM2pEFiZvZwsow,21014
16
+ geovisio/translations/da/LC_MESSAGES/messages.po,sha256=Yt1JSEG9e_ljgRvc0u3cknQ5ViU-eJq9XNQ__1Nz-Vs,29084
17
+ geovisio/translations/de/LC_MESSAGES/messages.mo,sha256=rO-g7WJC74c9MeuqSAuc3bNZX4G5Vz4O-sHILirjYYk,22590
18
+ geovisio/translations/de/LC_MESSAGES/messages.po,sha256=0LcBQjH0LpsHSypXo7kKQKZC0K_yPNyHBuEh5aF7lIk,30776
19
+ geovisio/translations/el/LC_MESSAGES/messages.mo,sha256=vy1jtEG6mLS5sYWPfQIr5U4XsZ21ZzSbsHAJHGQXZSY,433
20
+ geovisio/translations/el/LC_MESSAGES/messages.po,sha256=gDr-pDCsQGrCXBMBcDwlfsxcGWF1NIEqGrqPcZy65-4,17405
21
+ geovisio/translations/en/LC_MESSAGES/messages.mo,sha256=TdE2eywWyvkF0x0qsBPHju3TugQLLqtTk4ZqnAxuJHs,19953
22
+ geovisio/translations/en/LC_MESSAGES/messages.po,sha256=Q7j87mLUtyG-wrVvLQXlbTecW8p04a71FnN919UjZN8,27942
23
+ geovisio/translations/eo/LC_MESSAGES/messages.mo,sha256=3xtkpuSWl-Wb61cwlB7yK40_JgLMJNB5k1FCFuCfqM8,19330
24
+ geovisio/translations/eo/LC_MESSAGES/messages.po,sha256=Gd9n9ZalupqFAlI48y-VQ-XZumTVWnpEOy8-VXnR3Us,26977
25
+ geovisio/translations/es/LC_MESSAGES/messages.mo,sha256=ULQDhq4enQyjfAWxDq13BwHBPibg3Yt4ys6XrfQF5tM,19111
26
+ geovisio/translations/es/LC_MESSAGES/messages.po,sha256=NoyuXR_2iugWHLTcoUZSNLUSCpt8jMyl0FUD4p7N99w,26775
27
+ geovisio/translations/fi/LC_MESSAGES/messages.mo,sha256=6-WCesFiV00MkNM_Wpi7-D51DOZRNg_QOM2sL7-UPhA,626
28
+ geovisio/translations/fi/LC_MESSAGES/messages.po,sha256=UFT4YCfEazxLij8Ovk2vZqx55e2Yctbf_3xM5KDrXhw,14685
29
+ geovisio/translations/fr/LC_MESSAGES/messages.mo,sha256=2kjpCTUbrbAUfFJZ17kR5FAdY_LD5Bh9POzkQvo06-Q,21712
30
+ geovisio/translations/fr/LC_MESSAGES/messages.po,sha256=e-PEtl8S4Hjymh9-rv4R45RM3dq-C1M2hdF4CE5Nm6E,29384
31
+ geovisio/translations/hu/LC_MESSAGES/messages.mo,sha256=R-0QJl78CNJepsXi8uunlCA-QHhB0_t1Xj6e_EI_NI4,20156
32
+ geovisio/translations/hu/LC_MESSAGES/messages.po,sha256=Cs1EaEfVISyIsKxnK-f0gy0ocJdey5o-620mkvW1SAs,27472
33
+ geovisio/translations/it/LC_MESSAGES/messages.mo,sha256=jpTyt3Kv19Qu7EHGTzunZyXGMGsg3cg3_EBVxvNmuYA,22260
34
+ geovisio/translations/it/LC_MESSAGES/messages.po,sha256=Jw3y5igDkAVvTmJz0sYoopXvOBops5lRqdC6F-j2AXQ,30335
35
+ geovisio/translations/ja/LC_MESSAGES/messages.mo,sha256=5t8PzVwGf7ePX3mCQI65pGFOLzF2sQbMPm8svxkxNAE,426
36
+ geovisio/translations/ja/LC_MESSAGES/messages.po,sha256=a5S3Lceg47RFSuq7Lcytmu1ni0JGj8nPjWXdF1ZWRVs,18352
37
+ geovisio/translations/ko/LC_MESSAGES/messages.mo,sha256=eKuQS9zLcJ9s-DzbfR-QK2INBJL10jTIQ1kuSTdJ9Rg,426
38
+ geovisio/translations/ko/LC_MESSAGES/messages.po,sha256=uq19EZaeRB-obmE1hYnckA8T12JuuU3nXYyKaMR4tiU,17405
39
+ geovisio/translations/nl/LC_MESSAGES/messages.mo,sha256=aKM90Hp4Eh9vCQba_tlfjEWlhygLXWGq_SVYqBw9IA4,1592
40
+ geovisio/translations/nl/LC_MESSAGES/messages.po,sha256=m69xfphxpgfPOuUrBK51XrR8UFwqCEBZpnb_5B1mGOU,15302
41
+ geovisio/translations/pl/LC_MESSAGES/messages.mo,sha256=0RiGTq49esjtIrBomzeFwG6dWdAlDhJxIOv4AgNjyOQ,10083
42
+ geovisio/translations/pl/LC_MESSAGES/messages.po,sha256=JpN6yFqnpyqkMjU3YZLAMr65npQ7AybOLB3ARJoL94g,22192
43
+ geovisio/translations/zh_Hant/LC_MESSAGES/messages.mo,sha256=TmRUyfTGoBpU-2BE-nKjhwdr9r0ikDioVQU-JQ_ih90,431
44
+ geovisio/translations/zh_Hant/LC_MESSAGES/messages.po,sha256=LnnKlHy8t_54nNsLDBqC1eEPwPx49h1Um9mQj6l9hv0,18357
45
+ geovisio/utils/__init__.py,sha256=g4SWVoV73cSXjf5-5D9-HmyB5xKmHSuxxOGWnx7W3V0,71
46
+ geovisio/utils/auth.py,sha256=_vvkBTvjRXYlnyaHziNWJjiGKulomMqex-CDbv1dbKQ,13845
47
+ geovisio/utils/db.py,sha256=DFyCEB5-xTUo6sn79SYJCzuvlgFNDVyNJ48Mana5vPI,2625
48
+ geovisio/utils/excluded_areas.py,sha256=6f3wwsgNpJKxAXnHH8RKlktgHpsG-0QVNTWDDTFqPZ8,2585
49
+ geovisio/utils/extent.py,sha256=vzOHvbG6lpSNt7KrsaonBOx7Tz46S1J603gLbZvs36g,557
50
+ geovisio/utils/fields.py,sha256=sNAmrSJ4e-nqm0-LoyO3l4Zynb-Jy8swhwmL3UcDN_o,2129
51
+ geovisio/utils/filesystems.py,sha256=W_wH7TlvdEux_q4FP0XInxruxlbepFSEpJbPLO9Cnr4,4133
52
+ geovisio/utils/link.py,sha256=u9x4xJa57L1448neD7uPJuAA76_sFXVE0-9_zPW-bJM,490
53
+ geovisio/utils/model_query.py,sha256=PtvYCjKVygmicvqqYpCpEKWUIEvwdEG6QMh-JL5E8AQ,2031
54
+ geovisio/utils/params.py,sha256=s9kBPHm4gRhMx10SD7mOPdG0tR_n_O-g_rgL8Fife6s,630
55
+ geovisio/utils/pictures.py,sha256=cDDOABzZaTn98Bg8lYgoMlkTNklS9-y-qB-HXTcJ0YM,23092
56
+ geovisio/utils/reports.py,sha256=PgU0Td48WJg6XCq043he8Jif3WCA9nOTaGE0Yovo3h0,5626
57
+ geovisio/utils/semantics.py,sha256=bsPo4n0R0_pU5NdL7-dkx-bMYPhtyVq85z4nHNLptto,4572
58
+ geovisio/utils/sentry.py,sha256=Dw0rW2sNFbcpGojzpa-vjtJ5S7BH_XhsqfuGYBjHCG0,3759
59
+ geovisio/utils/sequences.py,sha256=S3OaMozzk9viYJMgHyzGYP1hZFsYjF8Ez2hXFTIBimI,25307
60
+ geovisio/utils/time.py,sha256=-hOs7FSx-8eiGmMQtTOrCoF8d_yMlj2306vfxJftEr8,642
61
+ geovisio/utils/tokens.py,sha256=tkihnnXqgQeIME_d12tC8PVrPN90A0i9k6UPEbgZ9TQ,3047
62
+ geovisio/utils/upload_set.py,sha256=P27ABINmMgNaFGMqd6rLLbeaOSddTz7Tchs2ByKDHeM,26478
63
+ geovisio/utils/website.py,sha256=wQosLHD-7_ONReJijKCjGUawKz1eCyBLWwkSWxd6ei8,1909
64
+ geovisio/web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
65
+ geovisio/web/annotations.py,sha256=TdivDOcVh83HRkBXBTxFD7J-VbZ1gnVesPfTNY-3uW4,653
66
+ geovisio/web/auth.py,sha256=-msYF5q2OUsa3rYH7H0vjclt6JEaPy9CHzjWmCCwZUU,7139
67
+ geovisio/web/collections.py,sha256=pt181nK3bTa1UYn9qpi3RBGH5h5I8VKMOb_Kfa-c-EU,46470
68
+ geovisio/web/configuration.py,sha256=tWZYxOoqI2MQwmuHk1I9J2DKzDqpLBVmWRDSsx18U7E,2177
69
+ geovisio/web/docs.py,sha256=RxMtyH_Urcr0YWPTZz1epMKr-iHRwRXBdRzgdhZzaWU,56054
70
+ geovisio/web/excluded_areas.py,sha256=5BNSZ0UqgFMtgvgrJ73eYGJXPJRnV-mGEs36WDRRxTk,13024
71
+ geovisio/web/items.py,sha256=1a_O5tes2NL4DHqDWsgJn_PQQYx4Ft6v4t7ibzkgtdw,61357
72
+ geovisio/web/map.py,sha256=DaigXevz4lL7WGjPFCsKbXvjdDevFj9gpH0-22JsJOI,25328
73
+ geovisio/web/pages.py,sha256=Hkc3KJFE6D38vGnkCK5WUBJ8KQemI1f1wGXpxeiOiNo,6632
74
+ geovisio/web/params.py,sha256=Ip2qFR2Z28OTP7Fe1EuineHKUMmlZ_u5dFW77MqdLD0,21071
75
+ geovisio/web/pictures.py,sha256=qbhgLsI6YtpFxXn1a3dzO66nnVrWglRZSXWmlfJr1tU,6394
76
+ geovisio/web/prepare.py,sha256=R10_xf6O9dmAAwOMC-vsaxgNTdc9BkDJLATqH6MKtCw,5620
77
+ geovisio/web/reports.py,sha256=8v9a4PMM9RsvSGadZEN2o5PTKG_TohjyMMEBfFeY13E,14123
78
+ geovisio/web/rss.py,sha256=NLUd2Or92tcKRaGUHAze6QMLWczHyzag9ybOzrA8djE,2962
79
+ geovisio/web/stac.py,sha256=1uoSUOgCxOdH4UQuUvt-0xJaPLtPcAD54WvQg0lvxwM,14850
80
+ geovisio/web/tokens.py,sha256=l7CAM0FQ6qAcoUhtIRysKc9Gndlji_wOMpkXLsPP1pI,9599
81
+ geovisio/web/upload_set.py,sha256=IkZrSosJ6LenqEsNFx4BOULZCUrcl8yIz_sO4B5xyeM,30671
82
+ geovisio/web/users.py,sha256=CB5fJxZyIUDtlQ-BKfV4zZtnlSGqgZehsMk2rlVTvHk,13038
83
+ geovisio/web/utils.py,sha256=kudTbV4Tgtkbd4oUWFTFpyWNINpxAa-VQNbxYEiR6pM,3640
84
+ geovisio/workers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
85
+ geovisio/workers/runner_pictures.py,sha256=Y4x345tp0Y3RAFnoYpcDhyg6dJS1OYx79XGGDIttcps,22898
86
+ geovisio-2.8.0.dist-info/LICENSE,sha256=iRFSz7MJ7_j4hh3hvIgzNbS2buy5NMva8lulaixd3IE,1069
87
+ geovisio-2.8.0.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
88
+ geovisio-2.8.0.dist-info/METADATA,sha256=xWj0uWtGYpQQ_Fmcp6XPd6MslTkeQHlVybq_7zSEjgs,4299
89
+ geovisio-2.8.0.dist-info/RECORD,,