geovisio 2.6.0__py3-none-any.whl → 2.7.1__py3-none-any.whl

geovisio/__init__.py

@@ -1,19 +1,34 @@
 """GeoVisio API - Main"""
 
-__version__ = "2.6.0"
+__version__ = "2.7.1"
 
 import os
-from flask import Flask, jsonify, stream_template, send_from_directory, redirect
+from flask import Flask, jsonify, stream_template, send_from_directory, redirect, request
 from flask.cli import with_appcontext
 from flask_cors import CORS
 from flask_compress import Compress
+from flask_babel import Babel
 from flasgger import Swagger
 import logging
 from logging.config import dictConfig
 
 from geovisio import db_migrations, config_app, admin_cli, errors, utils
-from geovisio.utils import filesystems, sentry
-from geovisio.web import auth, docs, pictures, stac, map, users, configuration, tokens, collections, items
+from geovisio.utils import db, filesystems, sentry
+from geovisio.web import (
+    auth,
+    docs,
+    pictures,
+    stac,
+    map,
+    users,
+    configuration,
+    tokens,
+    collections,
+    items,
+    upload_set,
+    reports,
+    excluded_areas,
+)
 from geovisio.workers import runner_pictures
 
 
@@ -42,6 +57,14 @@ LOGGING_CONFIG = {
 dictConfig(LOGGING_CONFIG)
 
 
+# Init i18n
+def get_locale():
+    try:
+        return request.accept_languages.best_match(["de", "fr", "en", "es", "hu"]) or "en"
+    except:
+        return "en"
+
+
 def create_app(test_config=None, app=None):
     """API launcher method"""
     #
@@ -54,6 +77,8 @@ def create_app(test_config=None, app=None):
 
     config_app.read_config(app, test_config)
     sentry.init(app)
+    db.create_db_pool(app)
+    Babel(app, locale_selector=get_locale)
 
     # Prepare filesystem
     createDirNoFailure(app.instance_path)
@@ -78,7 +103,8 @@ def create_app(test_config=None, app=None):
 
         app.wsgi_app = ProxyFix(app.wsgi_app, x_for=nb_proxies, x_proto=nb_proxies, x_host=nb_proxies, x_prefix=nb_proxies)
 
-    runner_pictures.background_processor.init_app(app)
+    # store the background processor in the app context
+    app.background_processor = runner_pictures.PictureBackgroundProcessor(app)
 
     #
     # List available routes/blueprints
@@ -92,6 +118,9 @@ def create_app(test_config=None, app=None):
     app.register_blueprint(users.bp)
     app.register_blueprint(configuration.bp)
     app.register_blueprint(tokens.bp)
+    app.register_blueprint(upload_set.bp)
+    app.register_blueprint(reports.bp)
+    app.register_blueprint(excluded_areas.bp)
 
     # Register CLI comands
     app.register_blueprint(admin_cli.bp, cli_group=None)
@@ -101,8 +130,8 @@ def create_app(test_config=None, app=None):
     def run_picture_worker():
         """Run a worker to process pictures after upload. Each worker use one thread, and several workers can be run in parallel"""
         logging.info("Running picture worker")
-        worker = runner_pictures.PictureProcessor(config=app.config, stop=False)
-        worker.process_next_pictures()
+        worker = runner_pictures.PictureProcessor(app=app, stop=False)
+        worker.process_jobs()
 
     #
     # API documentation

geovisio/admin_cli/cleanup.py

@@ -60,12 +60,12 @@ def cleanup(sequences=[], full=False, database=False, cache=False, permanentPics
         if database:
             log.info("Cleaning up database...")
             if allSequences:
-                conn.execute("DELETE FROM pictures_to_process")
+                conn.execute("DELETE FROM job_queue")
                 conn.execute("DELETE FROM sequences_pictures")
                 conn.execute("DELETE FROM sequences")
                 conn.execute("DELETE FROM pictures")
             else:
-                conn.execute("DELETE FROM pictures_to_process WHERE picture_id = ANY(%s)", [pics])
+                conn.execute("DELETE FROM job_queue WHERE picture_id = ANY(%s)", [pics])
                 conn.execute("DELETE FROM sequences_pictures WHERE seq_id = ANY(%s)", [sequences])
                 conn.execute("DELETE FROM sequences WHERE id = ANY(%s)", [sequences])
                 conn.execute("DELETE FROM pictures WHERE id = ANY(%s)", [pics])

geovisio/admin_cli/db.py

@@ -1,7 +1,6 @@
 from flask import Blueprint, current_app
 from flask.cli import with_appcontext
 import click
-import psycopg
 from geovisio import db_migrations
 from geovisio.utils import sequences
 
@@ -33,6 +32,4 @@ def rollback(all):
 @with_appcontext
 def refresh():
     """Refresh cached data (pictures_grid)"""
-    with psycopg.connect(current_app.config["DB_URL"]) as db:
-        sequences.update_pictures_grid(db)
-        db.commit()
+    sequences.update_pictures_grid()

geovisio/config_app.py

@@ -3,11 +3,23 @@ import os.path
 from urllib.parse import urlparse
 import datetime
 import logging
-from typing import Optional
+from typing import Optional, Dict
 import croniter
+from pydantic import BaseModel
+from pydantic.color import Color
+from pydantic.networks import HttpUrl
+import json
+
+
+class ApiSummary(BaseModel):
+    name: Dict[str, str] = {"en": "GeoVisio"}
+    description: Dict[str, str] = {"en": "The open source photo mapping solution"}
+    logo: HttpUrl = "https://gitlab.com/panoramax/gitlab-profile/-/raw/main/images/logo.svg"
+    color: Color = "#bf360c"
 
 
 class DefaultConfig:
+    API_SUMMARY = ApiSummary()
     API_VIEWER_PAGE = "viewer.html"
     API_MAIN_PAGE = "main.html"
     # we default we keep the session cookie 7 days, users would have to renew their loggin after this
@@ -24,6 +36,10 @@ class DefaultConfig:
     PICTURE_PROCESS_REFRESH_CRON = (
         "0 2 * * *"  # Background worker will refresh by default some stats at 2 o'clock in the night (local time of the server)
     )
+    DB_MIN_CNX = 0
+    DB_MAX_CNX = 10
+    DB_STATEMENT_TIMEOUT = 5 * 60 * 1000  # default statement timeout in ms (5mn)
+    API_ACCEPT_DUPLICATE = False
 
 
 def read_config(app, test_config):
@@ -46,7 +62,11 @@ def read_config(app, test_config):
         "DB_PASSWORD",
         "DB_NAME",
         "DB_CHECK_SCHEMA",
+        "DB_MIN_CNX",
+        "DB_MAX_CNX",
+        "DB_STATEMENT_TIMEOUT",
         # API
+        "API_SUMMARY",
         "API_BLUR_URL",
         "API_VIEWER_PAGE",
         "API_MAIN_PAGE",
@@ -56,6 +76,8 @@ def read_config(app, test_config):
         "API_DERIVATES_PICTURES_PUBLIC_URL",
         "API_PICTURES_LICENSE_SPDX_ID",
         "API_PICTURES_LICENSE_URL",
+        "API_ACCEPT_DUPLICATE",
+        "API_GIT_VERSION",
         # Picture process
         "PICTURE_PROCESS_DERIVATES_STRATEGY",
         "PICTURE_PROCESS_THREADS_LIMIT",
@@ -130,6 +152,18 @@ def read_config(app, test_config):
             f"Unknown picture derivates strategy: '{app.config['PICTURE_PROCESS_DERIVATES_STRATEGY']}'. Please set to one of ON_DEMAND, PREPROCESS"
         )
 
+    # Parse API summary
+    if not isinstance(app.config.get("API_SUMMARY"), ApiSummary):
+        try:
+            if isinstance(app.config.get("API_SUMMARY"), str):
+                app.config["API_SUMMARY"] = ApiSummary(**json.loads(app.config["API_SUMMARY"]))
+            elif isinstance(app.config.get("API_SUMMARY"), dict):
+                app.config["API_SUMMARY"] = ApiSummary(**app.config["API_SUMMARY"])
+            elif app.config.get("API_SUMMARY") is not None:
+                raise Exception("Value is not a JSON")
+        except Exception as e:
+            raise Exception("Parameter API_SUMMARY is not recognized") from e
+
     # Checks on front-end related variables
     templateFolder = os.path.join(app.root_path, app.template_folder)
     for pageParam in ["API_MAIN_PAGE", "API_VIEWER_PAGE"]:
@@ -169,6 +203,11 @@ def read_config(app, test_config):
     cron_val = app.config["PICTURE_PROCESS_REFRESH_CRON"]
     if not croniter.croniter.is_valid(cron_val):
         raise Exception(f"PICTURE_PROCESS_REFRESH_CRON should be a valid cron syntax, got '{cron_val}'")
+
+    app.config["API_ACCEPT_DUPLICATE"] = _read_bool(app.config, "API_ACCEPT_DUPLICATE")
+
+    app.config["DB_STATEMENT_TIMEOUT"] = int(app.config["DB_STATEMENT_TIMEOUT"])
+
     #
     # Add generated config vars
     #

geovisio/db_migrations.py

@@ -1,5 +1,6 @@
 import yoyo
 import psycopg
+from psycopg.sql import SQL, Identifier
 import os
 import sys
 
@@ -8,6 +9,26 @@ def update_db_schema(dbUrl, force=False):
     # Check if DB has its structure initialized
     with psycopg.connect(dbUrl) as conn:
         with conn.cursor() as cursor:
+            # Alert if database is not UTC
+            dbTz = cursor.execute("SHOW timezone").fetchone()[0]
+            isUtcOffset = cursor.execute(
+                "SELECT EXISTS(SELECT * FROM pg_timezone_names WHERE name = %s AND utc_offset='00:00:00' AND NOT is_dst)", [dbTz]
+            ).fetchone()[0]
+            if not isUtcOffset:
+                dbName = Identifier(dbUrl.split("/")[-1])
+                if force:
+                    cursor.execute(SQL("ALTER DATABASE {db} SET TIMEZONE TO 'UTC'").format(db=dbName))
+                else:
+                    raise Exception(
+                        f"""Database is not running at UTC timezone !
+
+Your database actually uses timezone \"{dbTz}\".
+Issues could happen if your database runs at a different timezone.
+You can set the database timezone using the following command:
+
+ALTER DATABASE {dbName.as_string()} SET TIMEZONE TO 'UTC';"""
+                    )
+
             picturesTableExists = cursor.execute("SELECT EXISTS(SELECT relname FROM pg_class WHERE relname = 'pictures')").fetchone()[0]
             yoyoExists = cursor.execute("SELECT EXISTS(SELECT relname FROM pg_class WHERE relname like '_yoyo_%')").fetchone()[0]
         conn.close()
@@ -22,7 +43,7 @@ def update_db_schema(dbUrl, force=False):
             handledMigrations = [
                 m for m in migrations if m.id in ["20221201_01_wpCGc-initial-schema", "20221201_02_ZG8AR-camera-information"]
             ]
-            print(f"Database migrated to use Yoyo tools...")
+            print("Database migrated to use Yoyo tools...")
             backend.mark_migrations(handledMigrations)
 
         migrationToApply = backend.to_apply(migrations)
@@ -69,7 +90,7 @@ def rollback_db_schema(dbUrl, rollback_all):
         if not rollback_all:
             migrations_to_rollback = migrations_to_rollback[:1]  # we only rollback the last one
         if len(migrations_to_rollback) > 0:
-            print(f"Starting rollback for migrations:")
+            print("Starting rollback for migrations:")
             for m in migrations_to_rollback:
                 print(f"  - {m.id}")
             backend.rollback_migrations(migrations_to_rollback)
@@ -79,7 +100,7 @@ def rollback_db_schema(dbUrl, rollback_all):
 
 
 def get_yoyo_backend(dbUrl):
-    dbUrl = dbUrl.replace("postgres://", "postgresql+psycopg://")  # force spycopg3 usage on yolo
+    dbUrl = dbUrl.replace("postgres://", "postgresql+psycopg://")  # force psycopg3 usage on yolo
     return yoyo.get_backend(dbUrl)
 
 

geovisio/templates/main.html

@@ -29,7 +29,7 @@
 
 	<script>
 		var baseUrl = window.location.href.replace(/\/$/, '');
-		var viewerUrl = "https://cdn.jsdelivr.net/npm/geovisio@~{{API_VERSION_MAJOR_MINOR}}/build";
+		var viewerUrl = "https://cdn.jsdelivr.net/npm/geovisio@~3.0/build";
 		function encodeHTML(html) {
 			return html
 				.replace(/</g, "&lt;")
@@ -42,22 +42,22 @@
 </head>
 <body>
 	<h1><img src="/static/img/logo_full.svg" style="width: 350px; max-width: 90%" alt="GeoVisio" /></h1>
-	<p>Simple 360° geolocated pictures hosting</p>
+	<p>{%trans%}Simple 360° geolocated pictures hosting{%endtrans%}</p>
 	<iframe src="/viewer" style="border: none; width: 95%; height: 300px"></iframe>
-	<p><small><a href="/viewer">Full page version</a></small></p>
+	<p><small><a href="/viewer">{%trans%}Full page version{%endtrans%}</a></small></p>
 
 	<hr />
 
-	<h2>Viewer</h2>
-	<h3>Embed pre-configured viewer</h3>
-	<p>Easiest way to have a working GeoVisio viewer on your website</p>
+	<h2>{%trans%}Viewer{%endtrans%}</h2>
+	<h3>{%trans%}Embed pre-configured viewer{%endtrans%}</h3>
+	<p>{%trans%}Easiest way to have a working GeoVisio viewer on your website{%endtrans%}</p>
 	<pre><code id="code-preconf"></code></pre>
 	<script>
 		document.getElementById("code-preconf").innerHTML = encodeHTML('<iframe \n\tsrc="'+baseUrl+'/viewer"\n\tstyle="border: none; width: 500px; height: 300px">\n</iframe>');
 	</script>
 
-	<h3>Use JS library</h3>
-	<p>A completely configurable viewer for your website</p>
+	<h3>{%trans%}Use JS library{%endtrans%}</h3>
+	<p>{%trans%}A completely configurable viewer for your website{%endtrans%}</p>
 	<pre><code id="code-js-html">
 	</code></pre>
 	<script>
@@ -79,15 +79,15 @@
 
 	<hr />
 
-	<h2>Links</h2>
+	<h2>{%trans%}Links{%endtrans%}</h2>
 	<p>
-		<a href="/viewer">Pictures viewer</a>
+		<a href="/viewer">{%trans%}Pictures viewer{%endtrans%}</a>
 		-
-		<a href="/api/docs/swagger">API docs</a>
+		<a href="/api/docs/swagger">{%trans%}API docs{%endtrans%}</a>
 		-
-		<a href="https://gitlab.com/panoramax/clients/web-viewer/-/tree/develop/docs">JS library docs</a>
+		<a href="https://gitlab.com/panoramax/clients/web-viewer/-/tree/develop/docs">{%trans%}JS library docs{%endtrans%}</a>
 		-
-		<a href="https://gitlab.com/geovisio">Repositories</a>
+		<a href="https://gitlab.com/panoramax">{%trans%}Repositories{%endtrans%}</a>
 	</p>
 </body>
 </html>

geovisio/templates/viewer.html

@@ -6,7 +6,7 @@
 	<link rel="shortcut icon" href="/favicon.ico" />
 	<link rel="icon" type="image/svg+xml" href="/static/img/favicon.svg" />
 	<title>GeoVisio</title>
-	<link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/geovisio@~{{API_VERSION_MAJOR_MINOR}}/build/index.css" />
+	<link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/geovisio@~3.0/build/index.css" />
 	<style>
 		#viewer {
 			position: absolute;
@@ -19,10 +19,10 @@
 </head>
 <body>
 	<div id="viewer">
-		<noscript>You need to enable JavaScript to run this app.</noscript>
+		<noscript>{%trans%}You need to enable JavaScript to run this app.{%endtrans%}</noscript>
 	</div>
 
-	<script src="https://cdn.jsdelivr.net/npm/geovisio@~{{API_VERSION_MAJOR_MINOR}}/build/index.js"></script>
+	<script src="https://cdn.jsdelivr.net/npm/geovisio@~3.0/build/index.js"></script>
 	<script>
 		var instance = new GeoVisio.default(
 			"viewer",