geovisio 2.5.0__py3-none-any.whl → 2.6.0__py3-none-any.whl

geovisio/__init__.py

@@ -1,6 +1,6 @@
 """GeoVisio API - Main"""
 
-__version__ = "2.5.0"
+__version__ = "2.6.0"
 
 import os
 from flask import Flask, jsonify, stream_template, send_from_directory, redirect
@@ -60,7 +60,8 @@ def create_app(test_config=None, app=None):
     app.config["FILESYSTEMS"] = filesystems.openFilesystemsFromConfig(app.config)
 
     # Check database connection and update its schema if needed
-    db_migrations.update_db_schema(app.config["DB_URL"])
+    if app.config.get("DB_CHECK_SCHEMA"):
+        db_migrations.update_db_schema(app.config["DB_URL"])
 
     if app.config.get("OAUTH_PROVIDER"):
         utils.auth.make_auth(app)

geovisio/admin_cli/__init__.py

@@ -58,8 +58,8 @@ def cleanup_cmd(sequencesids, full, database, cache, permanent_pictures):
 @bp.cli.command("process-sequences")
 @with_appcontext
 def process_sequences():
-    """Deprecated entry point, use https://gitlab.com/geovisio/cli to upload a sequence instead"""
-    logging.error("This function has been deprecated, use https://gitlab.com/geovisio/cli to upload a sequence instead.")
+    """Deprecated entry point, use https://gitlab.com/panoramax/clients/cli to upload a sequence instead"""
+    logging.error("This function has been deprecated, use https://gitlab.com/panoramax/clients/cli to upload a sequence instead.")
     logging.error(
         "To upload a sequence with this tool, install it with `pip install geovisio_cli`, then run:\ngeovisio upload --path <directory> --api-url <api-url>"
     )

geovisio/admin_cli/db.py

@@ -1,7 +1,9 @@
 from flask import Blueprint, current_app
 from flask.cli import with_appcontext
 import click
+import psycopg
 from geovisio import db_migrations
+from geovisio.utils import sequences
 
 bp = Blueprint("db", __name__)
 
@@ -25,3 +27,12 @@ def upgrade():
 def rollback(all):
     """Rollbacks the latest database migration"""
     db_migrations.rollback_db_schema(current_app.config["DB_URL"], all)
+
+
+@bp.cli.command("refresh")
+@with_appcontext
+def refresh():
+    """Refresh cached data (pictures_grid)"""
+    with psycopg.connect(current_app.config["DB_URL"]) as db:
+        sequences.update_pictures_grid(db)
+        db.commit()

geovisio/config_app.py

@@ -3,6 +3,8 @@ import os.path
 from urllib.parse import urlparse
 import datetime
 import logging
+from typing import Optional
+import croniter
 
 
 class DefaultConfig:
@@ -14,9 +16,14 @@ class DefaultConfig:
     PICTURE_PROCESS_DERIVATES_STRATEGY = "ON_DEMAND"
     API_BLUR_URL = None
     PICTURE_PROCESS_THREADS_LIMIT = 1
+    DB_CHECK_SCHEMA = True  # If True check the database schema, and do not start the api if not up to date
     API_PICTURES_LICENSE_SPDX_ID = None
     API_PICTURES_LICENSE_URL = None
     DEBUG_PICTURES_SKIP_FS_CHECKS_WITH_PUBLIC_URL = False
+    SESSION_COOKIE_HTTPONLY = False
+    PICTURE_PROCESS_REFRESH_CRON = (
+        "0 2 * * *"  # Background worker will refresh by default some stats at 2 o'clock in the night (local time of the server)
+    )
 
 
 def read_config(app, test_config):
@@ -38,6 +45,7 @@ def read_config(app, test_config):
         "DB_USERNAME",
         "DB_PASSWORD",
         "DB_NAME",
+        "DB_CHECK_SCHEMA",
         # API
         "API_BLUR_URL",
         "API_VIEWER_PAGE",
@@ -51,6 +59,7 @@ def read_config(app, test_config):
         # Picture process
         "PICTURE_PROCESS_DERIVATES_STRATEGY",
         "PICTURE_PROCESS_THREADS_LIMIT",
+        "PICTURE_PROCESS_REFRESH_CRON",
         # OAUTH
         "OAUTH_PROVIDER",
         "OAUTH_OIDC_URL",
@@ -106,6 +115,8 @@ def read_config(app, test_config):
 
         app.config["DB_URL"] = f"postgres://{username}:{passw}@{host}:{port}/{dbname}"
 
+    app.config["DB_CHECK_SCHEMA"] = _read_bool(app.config, "DB_CHECK_SCHEMA")
+
     if app.config.get("API_BLUR_URL") is not None and len(app.config.get("API_BLUR_URL")) > 0:
         try:
             urlparse(app.config.get("API_BLUR_URL"))
@@ -155,6 +166,9 @@ def read_config(app, test_config):
     if app.config.get("API_PICTURES_LICENSE_SPDX_ID") is None:
         app.config["API_PICTURES_LICENSE_SPDX_ID"] = "proprietary"
 
+    cron_val = app.config["PICTURE_PROCESS_REFRESH_CRON"]
+    if not croniter.croniter.is_valid(cron_val):
+        raise Exception(f"PICTURE_PROCESS_REFRESH_CRON should be a valid cron syntax, got '{cron_val}'")
     #
     # Add generated config vars
     #
@@ -164,6 +178,17 @@ def read_config(app, test_config):
     app.config["EXECUTOR_PROPAGATE_EXCEPTIONS"] = True  # propagate the excecutor's exceptions, to be able to trace them
 
 
+def _read_bool(config, value_name: str) -> Optional[bool]:
+    value = config.get(value_name)
+    if value is None:
+        return value
+    if type(value) == bool:
+        return value
+    if type(value) == str:
+        return value.lower() == "true"
+    raise Exception(f"Configuration {value_name} should either be a boolean or a string, got '{value}'")
+
+
 def _get_threads_limit(param: str) -> int:
     """Computes maximum thread limit depending on environment variables and available CPU.
 

geovisio/templates/main.html

@@ -67,7 +67,7 @@
 			+'<div id="viewer" style="width: 500px; height: 300px"></div>\n\n'
 			+'&lt;script>\n'
 			+'\t// All options available are listed here\n'
-			+'\t// https://gitlab.com/geovisio/web-viewer/-/blob/develop/docs/02_Usage.md\n'
+			+'\t// https://gitlab.com/panoramax/clients/web-viewer/-/blob/develop/docs/02_Usage.md\n'
 			+'\tvar instance = new GeoVisio.default(\n'
 			+'\t\t"viewer",\n'
 			+'\t\t"'+baseUrl+'/api",\n'
@@ -85,7 +85,7 @@
 		-
 		<a href="/api/docs/swagger">API docs</a>
 		-
-		<a href="https://gitlab.com/geovisio/web-viewer/-/tree/develop/docs">JS library docs</a>
+		<a href="https://gitlab.com/panoramax/clients/web-viewer/-/tree/develop/docs">JS library docs</a>
 		-
 		<a href="https://gitlab.com/geovisio">Repositories</a>
 	</p>

geovisio/utils/pictures.py

@@ -537,6 +537,7 @@ def insertNewPictureInDatabase(db, sequenceId, position, pictureBytes, associate
     lighterMetadata = dict(filter(lambda v: v[0] not in ["ts", "heading", "lon", "lat", "exif"], metadata.items()))
     if lighterMetadata.get("tagreader_warnings") is not None and len(lighterMetadata["tagreader_warnings"]) == 0:
         del lighterMetadata["tagreader_warnings"]
+    lighterMetadata["tz"] = metadata["ts"].tzname()
 
     exif = cleanupExif(metadata["exif"])
 
@@ -545,11 +546,11 @@ def insertNewPictureInDatabase(db, sequenceId, position, pictureBytes, associate
         picId = db.execute(
             """
 			INSERT INTO pictures (ts, heading, metadata, geom, account_id, exif)
-			VALUES (to_timestamp(%s), %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326), %s, %s)
+			VALUES (%s, %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326), %s, %s)
 			RETURNING id
 		""",
             (
-                metadata["ts"],
+                metadata["ts"].isoformat(),
                 metadata["heading"],
                 Jsonb(lighterMetadata),
                 metadata["lon"],
@@ -603,7 +604,7 @@ def insertNewPictureInDatabase(db, sequenceId, position, pictureBytes, associate
 
 
 # Note: we don't want to store and expose exif binary fields as they are difficult to use and take a lot of storage in the database (~20% for maker notes only)
-# This list has been queried from real data (cf [this comment](https://gitlab.com/geovisio/api/-/merge_requests/241#note_1790580636)).
+# This list has been queried from real data (cf [this comment](https://gitlab.com/panoramax/server/api/-/merge_requests/241#note_1790580636)).
 # Update this list (and do a sql migration) if new binary fields are added
 BLACK_LISTED_BINARY_EXIF_FIELDS = set(
     [

geovisio/utils/sequences.py

@@ -12,7 +12,9 @@ from enum import Enum
 from geovisio.utils.fields import FieldMapping, SortBy, SQLDirection, BBox, Bounds
 from geopic_tag_reader import reader
 from pathlib import PurePath
-from geovisio import errors
+from geovisio import errors, utils
+import logging
+import sentry_sdk
 
 
 def createSequence(metadata, accountId) -> str:
@@ -138,37 +140,29 @@ def get_collections(request: CollectionsRequest) -> Collections:
         with conn.cursor() as cursor:
             sqlSequencesRaw = SQL(
                 """
-                SELECT * FROM (
-                    SELECT
-                        s.id,
-                        s.status,
-                        s.metadata->>'title' AS name,
-                        s.inserted_at AS created,
-                        s.updated_at AS updated,
-                        ST_XMin(s.bbox) AS minx,
-                        ST_YMin(s.bbox) AS miny,
-                        ST_XMax(s.bbox) AS maxx,
-                        ST_YMax(s.bbox) AS maxy,
-                        accounts.name AS account_name,
-                        ST_X(ST_PointN(s.geom, 1)) AS x1,
-                        ST_Y(ST_PointN(s.geom, 1)) AS y1,
-                        {status},
-                        s.computed_capture_date AS datetime
-                    FROM sequences s
-                    LEFT JOIN accounts on s.account_id = accounts.id
-                    WHERE {filter}
-                    ORDER BY {order1}
-                    LIMIT {limit}
-                ) s
-                LEFT JOIN LATERAL (
-                    SELECT MIN(p.ts) as mints,
-                            MAX(p.ts) as maxts,
-                            COUNT(p.*) AS nbpic
-                    FROM sequences_pictures sp
-                    JOIN pictures p ON sp.pic_id = p.id
-                    WHERE {pic_filter}
-                    GROUP BY sp.seq_id
-                ) sub ON true
+                SELECT
+                    s.id,
+                    s.status,
+                    s.metadata->>'title' AS name,
+                    s.inserted_at AS created,
+                    s.updated_at AS updated,
+                    ST_XMin(s.bbox) AS minx,
+                    ST_YMin(s.bbox) AS miny,
+                    ST_XMax(s.bbox) AS maxx,
+                    ST_YMax(s.bbox) AS maxy,
+                    accounts.name AS account_name,
+                    ST_X(ST_PointN(s.geom, 1)) AS x1,
+                    ST_Y(ST_PointN(s.geom, 1)) AS y1,
+                    s.min_picture_ts AS mints,
+                    s.max_picture_ts AS maxts,
+                    s.nb_pictures AS nbpic,
+                    {status},
+                    s.computed_capture_date AS datetime
+                FROM sequences s
+                LEFT JOIN accounts on s.account_id = accounts.id
+                WHERE {filter}
+                ORDER BY {order1}
+                LIMIT {limit}
             """
             )
             sqlSequences = sqlSequencesRaw.format(
@@ -480,3 +474,33 @@ def update_headings(
         ),  # lots of camera have heading set to 0 for unset heading, so we recompute the heading when it's 0 too, even if this could be a valid value
         {"seq": sequenceId, "diff": relativeHeading, "account": editingAccount},
     )
+
+
+def update_pictures_grid(db) -> bool:
+    """Refreshes the pictures_grid materialized view for an up-to-date view of pictures availability on map.
+
+    Note: the transaction is not commited at the end, you need to commit it or use an autocommit connection.
+
+    Parameters
+    ----------
+    db : psycopg.Connection
+            Database connection
+
+    Returns
+    -------
+    bool : True if the view has been updated else False
+    """
+    logger = logging.getLogger("geovisio.picture_grid")
+    with db.transaction():
+        try:
+            db.execute("SELECT refreshed_at FROM refresh_database FOR UPDATE NOWAIT").fetchone()
+        except psycopg.errors.LockNotAvailable:
+            logger.info("Database refresh already in progress, nothing to do")
+            return False
+
+        with sentry_sdk.start_span(description="Refreshing database") as span:
+            with utils.time.log_elapsed(f"Refreshing database", logger=logger):
+                logger.info("Refreshing database")
+                db.execute("UPDATE refresh_database SET refreshed_at = NOW()")
+                db.execute("REFRESH MATERIALIZED VIEW pictures_grid")
+    return True

geovisio/web/auth.py

@@ -3,8 +3,9 @@ from flask import current_app, url_for, session, redirect, request, jsonify
 import psycopg
 from typing import Any
 from urllib.parse import quote
-from geovisio import utils
+from geovisio import utils, errors
 from geovisio.utils.auth import Account, ACCOUNT_KEY
+from authlib.integrations.base_client.errors import MismatchingStateError
 
 bp = flask.Blueprint("auth", __name__, url_prefix="/api/auth")
 
@@ -52,7 +53,19 @@ def auth():
                     schema:
                         type: string
     """
-    tokenResponse = utils.auth.oauth_provider.client.authorize_access_token()
+    try:
+        tokenResponse = utils.auth.oauth_provider.client.authorize_access_token()
+    except MismatchingStateError as e:
+        raise errors.InternalError(
+            "Impossible to finish authentication flow",
+            payload={
+                "details": {
+                    "error": str(e),
+                    "tips": "You can try to clear your cookies and retry. If the problem persists, contact your instance administrator.",
+                }
+            },
+            status_code=403,
+        )
 
     oauth_info = utils.auth.oauth_provider.get_user_oauth_info(tokenResponse)
     with psycopg.connect(current_app.config["DB_URL"]) as conn:

geovisio/web/collections.py

@@ -340,19 +340,19 @@ def getCollection(collectionId):
                     s.inserted_at AS created,
                     s.updated_at AS updated,
                     s.current_sort AS current_sort,
-                    a.*
+                    a.*,
+                    min_picture_ts AS mints,
+                    max_picture_ts AS maxts,
+                    nb_pictures AS nbpic
                 FROM sequences s
                 JOIN accounts ON s.account_id = accounts.id, (
                     SELECT
-                        MIN(ts) as mints,
-                        MAX(ts) as maxts,
                         array_agg(DISTINCT jsonb_build_object(
                             'make', metadata->>'make',
                             'model', metadata->>'model',
                             'focal_length', metadata->>'focal_length',
                             'field_of_view', metadata->>'field_of_view'
-                        )) AS metas,
-                        COUNT(*) AS nbpic
+                        )) AS metas
                     FROM pictures p
                     JOIN sequences_pictures sp ON sp.seq_id = %(id)s AND sp.pic_id = p.id
                 ) a
@@ -770,6 +770,18 @@ def getCollectionImportStatus(collectionId):
     with psycopg.connect(current_app.config["DB_URL"], row_factory=dict_row) as conn:
         with conn.cursor() as cursor:
             sequence_status = cursor.execute(
+                SQL(
+                    """SELECT status 
+                    FROM sequences
+                    WHERE id = %(seq_id)s
+                        AND (status != 'hidden' OR account_id = %(account)s)-- show deleted sequence here"""
+                ),
+                params,
+            ).fetchone()
+            if sequence_status is None:
+                raise errors.InvalidAPIUsage("Sequence doesn't exists", status_code=404)
+
+            pics_status = cursor.execute(
                 """WITH
 pic_jobs_stats AS (
     SELECT
@@ -796,47 +808,37 @@ pic_jobs_stats AS (
     pic_jobs_stats.nb_errors,
     pic_jobs_stats.last_job_finished_at
     FROM sequences s
-    LEFT JOIN sequences_pictures sp ON sp.seq_id = s.id
-    LEFT JOIN pictures p ON sp.pic_id = p.id
+    JOIN sequences_pictures sp ON sp.seq_id = s.id
+    JOIN pictures p ON sp.pic_id = p.id
     LEFT JOIN pic_jobs_stats ON pic_jobs_stats.picture_id = p.id
     WHERE
                 s.id = %(seq_id)s
                 AND (p IS NULL OR p.status != 'hidden' OR p.account_id = %(account)s)
-                AND (s.status != 'hidden' OR s.account_id = %(account)s) -- show deleted sequence here
     ORDER BY s.id, sp.rank
 )
-SELECT json_build_object(
-        'status', s.status,
-        'items', json_agg(
-                    json_strip_nulls(
-                        json_build_object(
-                            'id', i.id,
-                            -- status is a bit deprecated, we'll split this field in more fields (like `processing_in_progress`, `hidden`, ...)
-                            -- but we maintain it for retrocompatibility
-                            'status', CASE 
-                                    WHEN i.is_job_running IS TRUE THEN 'preparing' 
-                                    WHEN i.last_job_error IS NOT NULL THEN 'broken' 
-                                    ELSE i.status
-                                END, 
-                            'processing_in_progress', i.is_job_running,
-                            'process_error', i.last_job_error,
-                            'nb_errors', i.nb_errors,
-                            'processed_at', i.last_job_finished_at,
-                            'rank', i.rank
-                        )
-                )
+SELECT json_strip_nulls(
+        json_build_object(
+            'id', i.id,
+            -- status is a bit deprecated, we'll split this field in more fields (like `processing_in_progress`, `hidden`, ...)
+            -- but we maintain it for retrocompatibility
+            'status', CASE 
+                    WHEN i.is_job_running IS TRUE THEN 'preparing' 
+                    WHEN i.last_job_error IS NOT NULL THEN 'broken' 
+                    ELSE i.status
+                END, 
+            'processing_in_progress', i.is_job_running,
+            'process_error', i.last_job_error,
+            'nb_errors', i.nb_errors,
+            'processed_at', i.last_job_finished_at,
+            'rank', i.rank
         )
-) AS sequence
-FROM items i
-JOIN sequences s on i.seq_id = s.id
-GROUP by s.id;""",
+    ) as pic_status
+FROM items i;""",
                 params,
             ).fetchall()
+            pics = [p["pic_status"] for p in pics_status if len(p["pic_status"]) > 0]
 
-            if len(sequence_status) == 0:
-                raise errors.InvalidAPIUsage("Sequence doesn't exists", status_code=404)
-
-            return sequence_status[0]["sequence"]
+            return {"status": sequence_status["status"], "items": pics}
 
 
 @bp.route("/users/<uuid:userId>/collection")

geovisio/web/docs.py

@@ -120,6 +120,50 @@ API_CONFIG = {
             "STACItemSearchBody": {
                 "$ref": f"https://api.stacspec.org/v{utils.STAC_VERSION}/item-search/openapi.yaml#/components/schemas/searchBody"
             },
+            "MapLibreStyleJSON": {
+                "type": "object",
+                "description": """
+MapLibre Style JSON, see https://maplibre.org/maplibre-style-spec/ for reference.
+
+Source ID is either \"geovisio\" or \"geovisio_\{userId\}\".
+
+Layers ID are \"geovisio_grid\", \"geovisio_sequences\" and \"geovisio_pictures\", or with user UUID included (\"geovisio_\{userId\}_sequences\" and \"geovisio_\{userId\}_pictures\").
+
+Note that you may not rely only on these ID that could change through time.
+""",
+                "properties": {
+                    "version": {"type": "integer", "example": 8},
+                    "name": {"type": "string", "example": "GeoVisio Vector Tiles"},
+                    "sources": {
+                        "type": "object",
+                        "properties": {
+                            "geovisio": {
+                                "type": "object",
+                                "properties": {
+                                    "type": {"type": "string", "example": "vector"},
+                                    "minzoom": {"type": "integer", "example": "0"},
+                                    "maxzoom": {"type": "integer", "example": "15"},
+                                    "tiles": {"type": "array", "items": {"type": "string"}},
+                                },
+                            }
+                        },
+                    },
+                    "layers": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "id": {"type": "string"},
+                                "source": {"type": "string"},
+                                "source-layer": {"type": "string"},
+                                "type": {"type": "string"},
+                                "paint": {"type": "object"},
+                                "layout": {"type": "object"},
+                            },
+                        },
+                    },
+                },
+            },
             "GeoVisioLanding": {
                 "allOf": [
                     {"$ref": "#/components/schemas/STACLanding"},
@@ -312,6 +356,11 @@ If unset, sort order is unchanged.
                             "properties": {
                                 "type": "object",
                                 "properties": {
+                                    "datetimetz": {
+                                        "type": "string",
+                                        "format": "date-time",
+                                        "title": "Date & time with original timezone information",
+                                    },
                                     "geovisio:status": {"$ref": "#/components/schemas/GeoVisioItemStatus"},
                                     "geovisio:producer": {"type": "string"},
                                     "geovisio:image": {"type": "string", "format": "uri"},
@@ -558,7 +607,7 @@ Available properties are:
                     },
                 },
             },
-            "JWToken": {
+            "GeoVisioEncodedToken": {
                 "type": "object",
                 "properties": {
                     "id": {"type": "string"},
@@ -572,7 +621,7 @@ Available properties are:
             },
             "JWTokenClaimable": {
                 "allOf": [
-                    {"$ref": "#/components/schemas/JWToken"},
+                    {"$ref": "#/components/schemas/GeoVisioEncodedToken"},
                     {
                         "type": "object",
                         "properties": {