geovisio 2.5.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

geovisio/__init__.py

@@ -1,19 +1,34 @@
 """GeoVisio API - Main"""
 
-__version__ = "2.5.0"
+__version__ = "2.7.0"
 
 import os
-from flask import Flask, jsonify, stream_template, send_from_directory, redirect
+from flask import Flask, jsonify, stream_template, send_from_directory, redirect, request
 from flask.cli import with_appcontext
 from flask_cors import CORS
 from flask_compress import Compress
+from flask_babel import Babel
 from flasgger import Swagger
 import logging
 from logging.config import dictConfig
 
 from geovisio import db_migrations, config_app, admin_cli, errors, utils
-from geovisio.utils import filesystems, sentry
-from geovisio.web import auth, docs, pictures, stac, map, users, configuration, tokens, collections, items
+from geovisio.utils import db, filesystems, sentry
+from geovisio.web import (
+    auth,
+    docs,
+    pictures,
+    stac,
+    map,
+    users,
+    configuration,
+    tokens,
+    collections,
+    items,
+    upload_set,
+    reports,
+    excluded_areas,
+)
 from geovisio.workers import runner_pictures
 
 
@@ -42,6 +57,14 @@ LOGGING_CONFIG = {
 dictConfig(LOGGING_CONFIG)
 
 
+# Init i18n
+def get_locale():
+    try:
+        return request.accept_languages.best_match(["de", "fr", "en"]) or "en"
+    except:
+        return "en"
+
+
 def create_app(test_config=None, app=None):
     """API launcher method"""
     #
@@ -54,13 +77,16 @@ def create_app(test_config=None, app=None):
 
     config_app.read_config(app, test_config)
     sentry.init(app)
+    db.create_db_pool(app)
+    Babel(app, locale_selector=get_locale)
 
     # Prepare filesystem
     createDirNoFailure(app.instance_path)
     app.config["FILESYSTEMS"] = filesystems.openFilesystemsFromConfig(app.config)
 
     # Check database connection and update its schema if needed
-    db_migrations.update_db_schema(app.config["DB_URL"])
+    if app.config.get("DB_CHECK_SCHEMA"):
+        db_migrations.update_db_schema(app.config["DB_URL"])
 
     if app.config.get("OAUTH_PROVIDER"):
         utils.auth.make_auth(app)
@@ -77,7 +103,8 @@ def create_app(test_config=None, app=None):
 
         app.wsgi_app = ProxyFix(app.wsgi_app, x_for=nb_proxies, x_proto=nb_proxies, x_host=nb_proxies, x_prefix=nb_proxies)
 
-    runner_pictures.background_processor.init_app(app)
+    # store the background processor in the app context
+    app.background_processor = runner_pictures.PictureBackgroundProcessor(app)
 
     #
     # List available routes/blueprints
@@ -91,6 +118,9 @@ def create_app(test_config=None, app=None):
     app.register_blueprint(users.bp)
     app.register_blueprint(configuration.bp)
     app.register_blueprint(tokens.bp)
+    app.register_blueprint(upload_set.bp)
+    app.register_blueprint(reports.bp)
+    app.register_blueprint(excluded_areas.bp)
 
     # Register CLI comands
     app.register_blueprint(admin_cli.bp, cli_group=None)
@@ -100,8 +130,8 @@ def create_app(test_config=None, app=None):
     def run_picture_worker():
         """Run a worker to process pictures after upload. Each worker use one thread, and several workers can be run in parallel"""
         logging.info("Running picture worker")
-        worker = runner_pictures.PictureProcessor(config=app.config, stop=False)
-        worker.process_next_pictures()
+        worker = runner_pictures.PictureProcessor(app=app, stop=False)
+        worker.process_jobs()
 
     #
     # API documentation

geovisio/admin_cli/__init__.py

@@ -58,8 +58,8 @@ def cleanup_cmd(sequencesids, full, database, cache, permanent_pictures):
 @bp.cli.command("process-sequences")
 @with_appcontext
 def process_sequences():
-    """Deprecated entry point, use https://gitlab.com/geovisio/cli to upload a sequence instead"""
-    logging.error("This function has been deprecated, use https://gitlab.com/geovisio/cli to upload a sequence instead.")
+    """Deprecated entry point, use https://gitlab.com/panoramax/clients/cli to upload a sequence instead"""
+    logging.error("This function has been deprecated, use https://gitlab.com/panoramax/clients/cli to upload a sequence instead.")
     logging.error(
         "To upload a sequence with this tool, install it with `pip install geovisio_cli`, then run:\ngeovisio upload --path <directory> --api-url <api-url>"
     )

geovisio/admin_cli/db.py

@@ -2,6 +2,7 @@ from flask import Blueprint, current_app
 from flask.cli import with_appcontext
 import click
 from geovisio import db_migrations
+from geovisio.utils import sequences
 
 bp = Blueprint("db", __name__)
 
@@ -25,3 +26,10 @@ def upgrade():
 def rollback(all):
     """Rollbacks the latest database migration"""
     db_migrations.rollback_db_schema(current_app.config["DB_URL"], all)
+
+
+@bp.cli.command("refresh")
+@with_appcontext
+def refresh():
+    """Refresh cached data (pictures_grid)"""
+    sequences.update_pictures_grid()

geovisio/config_app.py

@@ -3,9 +3,23 @@ import os.path
 from urllib.parse import urlparse
 import datetime
 import logging
+from typing import Optional, Dict
+import croniter
+from pydantic import BaseModel
+from pydantic.color import Color
+from pydantic.networks import HttpUrl
+import json
+
+
+class ApiSummary(BaseModel):
+    name: Dict[str, str] = {"en": "GeoVisio"}
+    description: Dict[str, str] = {"en": "The open source photo mapping solution"}
+    logo: HttpUrl = "https://gitlab.com/panoramax/gitlab-profile/-/raw/main/images/logo.svg"
+    color: Color = "#bf360c"
 
 
 class DefaultConfig:
+    API_SUMMARY = ApiSummary()
     API_VIEWER_PAGE = "viewer.html"
     API_MAIN_PAGE = "main.html"
     # we default we keep the session cookie 7 days, users would have to renew their loggin after this
@@ -14,9 +28,18 @@ class DefaultConfig:
     PICTURE_PROCESS_DERIVATES_STRATEGY = "ON_DEMAND"
     API_BLUR_URL = None
     PICTURE_PROCESS_THREADS_LIMIT = 1
+    DB_CHECK_SCHEMA = True  # If True check the database schema, and do not start the api if not up to date
     API_PICTURES_LICENSE_SPDX_ID = None
     API_PICTURES_LICENSE_URL = None
     DEBUG_PICTURES_SKIP_FS_CHECKS_WITH_PUBLIC_URL = False
+    SESSION_COOKIE_HTTPONLY = False
+    PICTURE_PROCESS_REFRESH_CRON = (
+        "0 2 * * *"  # Background worker will refresh by default some stats at 2 o'clock in the night (local time of the server)
+    )
+    DB_MIN_CNX = 0
+    DB_MAX_CNX = 10
+    DB_STATEMENT_TIMEOUT = 5 * 60 * 1000  # default statement timeout in ms (5mn)
+    API_ACCEPT_DUPLICATE = False
 
 
 def read_config(app, test_config):
@@ -38,7 +61,12 @@ def read_config(app, test_config):
         "DB_USERNAME",
         "DB_PASSWORD",
         "DB_NAME",
+        "DB_CHECK_SCHEMA",
+        "DB_MIN_CNX",
+        "DB_MAX_CNX",
+        "DB_STATEMENT_TIMEOUT",
         # API
+        "API_SUMMARY",
         "API_BLUR_URL",
         "API_VIEWER_PAGE",
         "API_MAIN_PAGE",
@@ -48,9 +76,12 @@ def read_config(app, test_config):
         "API_DERIVATES_PICTURES_PUBLIC_URL",
         "API_PICTURES_LICENSE_SPDX_ID",
         "API_PICTURES_LICENSE_URL",
+        "API_ACCEPT_DUPLICATE",
+        "API_GIT_VERSION",
         # Picture process
         "PICTURE_PROCESS_DERIVATES_STRATEGY",
         "PICTURE_PROCESS_THREADS_LIMIT",
+        "PICTURE_PROCESS_REFRESH_CRON",
         # OAUTH
         "OAUTH_PROVIDER",
         "OAUTH_OIDC_URL",
@@ -106,6 +137,8 @@ def read_config(app, test_config):
 
         app.config["DB_URL"] = f"postgres://{username}:{passw}@{host}:{port}/{dbname}"
 
+    app.config["DB_CHECK_SCHEMA"] = _read_bool(app.config, "DB_CHECK_SCHEMA")
+
     if app.config.get("API_BLUR_URL") is not None and len(app.config.get("API_BLUR_URL")) > 0:
         try:
             urlparse(app.config.get("API_BLUR_URL"))
@@ -119,6 +152,18 @@ def read_config(app, test_config):
             f"Unknown picture derivates strategy: '{app.config['PICTURE_PROCESS_DERIVATES_STRATEGY']}'. Please set to one of ON_DEMAND, PREPROCESS"
         )
 
+    # Parse API summary
+    if not isinstance(app.config.get("API_SUMMARY"), ApiSummary):
+        try:
+            if isinstance(app.config.get("API_SUMMARY"), str):
+                app.config["API_SUMMARY"] = ApiSummary(**json.loads(app.config["API_SUMMARY"]))
+            elif isinstance(app.config.get("API_SUMMARY"), dict):
+                app.config["API_SUMMARY"] = ApiSummary(**app.config["API_SUMMARY"])
+            elif app.config.get("API_SUMMARY") is not None:
+                raise Exception("Value is not a JSON")
+        except Exception as e:
+            raise Exception("Parameter API_SUMMARY is not recognized") from e
+
     # Checks on front-end related variables
     templateFolder = os.path.join(app.root_path, app.template_folder)
     for pageParam in ["API_MAIN_PAGE", "API_VIEWER_PAGE"]:
@@ -155,6 +200,14 @@ def read_config(app, test_config):
     if app.config.get("API_PICTURES_LICENSE_SPDX_ID") is None:
         app.config["API_PICTURES_LICENSE_SPDX_ID"] = "proprietary"
 
+    cron_val = app.config["PICTURE_PROCESS_REFRESH_CRON"]
+    if not croniter.croniter.is_valid(cron_val):
+        raise Exception(f"PICTURE_PROCESS_REFRESH_CRON should be a valid cron syntax, got '{cron_val}'")
+
+    app.config["API_ACCEPT_DUPLICATE"] = _read_bool(app.config, "API_ACCEPT_DUPLICATE")
+
+    app.config["DB_STATEMENT_TIMEOUT"] = int(app.config["DB_STATEMENT_TIMEOUT"])
+
     #
     # Add generated config vars
     #
@@ -164,6 +217,17 @@ def read_config(app, test_config):
     app.config["EXECUTOR_PROPAGATE_EXCEPTIONS"] = True  # propagate the excecutor's exceptions, to be able to trace them
 
 
+def _read_bool(config, value_name: str) -> Optional[bool]:
+    value = config.get(value_name)
+    if value is None:
+        return value
+    if type(value) == bool:
+        return value
+    if type(value) == str:
+        return value.lower() == "true"
+    raise Exception(f"Configuration {value_name} should either be a boolean or a string, got '{value}'")
+
+
 def _get_threads_limit(param: str) -> int:
     """Computes maximum thread limit depending on environment variables and available CPU.
 

geovisio/db_migrations.py

@@ -1,5 +1,6 @@
 import yoyo
 import psycopg
+from psycopg.sql import SQL, Identifier
 import os
 import sys
 
@@ -8,6 +9,26 @@ def update_db_schema(dbUrl, force=False):
     # Check if DB has its structure initialized
     with psycopg.connect(dbUrl) as conn:
         with conn.cursor() as cursor:
+            # Alert if database is not UTC
+            dbTz = cursor.execute("SHOW timezone").fetchone()[0]
+            isUtcOffset = cursor.execute(
+                "SELECT EXISTS(SELECT * FROM pg_timezone_names WHERE name = %s AND utc_offset='00:00:00' AND NOT is_dst)", [dbTz]
+            ).fetchone()[0]
+            if not isUtcOffset:
+                dbName = Identifier(dbUrl.split("/")[-1])
+                if force:
+                    cursor.execute(SQL("ALTER DATABASE {db} SET TIMEZONE TO 'UTC'").format(db=dbName))
+                else:
+                    raise Exception(
+                        f"""Database is not running at UTC timezone !
+
+Your database actually uses timezone \"{dbTz}\".
+Issues could happen if your database runs at a different timezone.
+You can set the database timezone using the following command:
+
+ALTER DATABASE {dbName.as_string()} SET TIMEZONE TO 'UTC';"""
+                    )
+
             picturesTableExists = cursor.execute("SELECT EXISTS(SELECT relname FROM pg_class WHERE relname = 'pictures')").fetchone()[0]
             yoyoExists = cursor.execute("SELECT EXISTS(SELECT relname FROM pg_class WHERE relname like '_yoyo_%')").fetchone()[0]
         conn.close()
@@ -22,7 +43,7 @@ def update_db_schema(dbUrl, force=False):
             handledMigrations = [
                 m for m in migrations if m.id in ["20221201_01_wpCGc-initial-schema", "20221201_02_ZG8AR-camera-information"]
             ]
-            print(f"Database migrated to use Yoyo tools...")
+            print("Database migrated to use Yoyo tools...")
             backend.mark_migrations(handledMigrations)
 
         migrationToApply = backend.to_apply(migrations)
@@ -69,7 +90,7 @@ def rollback_db_schema(dbUrl, rollback_all):
         if not rollback_all:
             migrations_to_rollback = migrations_to_rollback[:1]  # we only rollback the last one
         if len(migrations_to_rollback) > 0:
-            print(f"Starting rollback for migrations:")
+            print("Starting rollback for migrations:")
             for m in migrations_to_rollback:
                 print(f"  - {m.id}")
             backend.rollback_migrations(migrations_to_rollback)
@@ -79,7 +100,7 @@ def rollback_db_schema(dbUrl, rollback_all):
 
 
 def get_yoyo_backend(dbUrl):
-    dbUrl = dbUrl.replace("postgres://", "postgresql+psycopg://")  # force spycopg3 usage on yolo
+    dbUrl = dbUrl.replace("postgres://", "postgresql+psycopg://")  # force psycopg3 usage on yolo
     return yoyo.get_backend(dbUrl)
 
 

geovisio/templates/main.html

@@ -29,7 +29,7 @@
 
 	<script>
 		var baseUrl = window.location.href.replace(/\/$/, '');
-		var viewerUrl = "https://cdn.jsdelivr.net/npm/geovisio@~{{API_VERSION_MAJOR_MINOR}}/build";
+		var viewerUrl = "https://cdn.jsdelivr.net/npm/geovisio@~3.0/build";
 		function encodeHTML(html) {
 			return html
 				.replace(/</g, "&lt;")
@@ -42,22 +42,22 @@
 </head>
 <body>
 	<h1><img src="/static/img/logo_full.svg" style="width: 350px; max-width: 90%" alt="GeoVisio" /></h1>
-	<p>Simple 360° geolocated pictures hosting</p>
+	<p>{%trans%}Simple 360° geolocated pictures hosting{%endtrans%}</p>
 	<iframe src="/viewer" style="border: none; width: 95%; height: 300px"></iframe>
-	<p><small><a href="/viewer">Full page version</a></small></p>
+	<p><small><a href="/viewer">{%trans%}Full page version{%endtrans%}</a></small></p>
 
 	<hr />
 
-	<h2>Viewer</h2>
-	<h3>Embed pre-configured viewer</h3>
-	<p>Easiest way to have a working GeoVisio viewer on your website</p>
+	<h2>{%trans%}Viewer{%endtrans%}</h2>
+	<h3>{%trans%}Embed pre-configured viewer{%endtrans%}</h3>
+	<p>{%trans%}Easiest way to have a working GeoVisio viewer on your website{%endtrans%}</p>
 	<pre><code id="code-preconf"></code></pre>
 	<script>
 		document.getElementById("code-preconf").innerHTML = encodeHTML('<iframe \n\tsrc="'+baseUrl+'/viewer"\n\tstyle="border: none; width: 500px; height: 300px">\n</iframe>');
 	</script>
 
-	<h3>Use JS library</h3>
-	<p>A completely configurable viewer for your website</p>
+	<h3>{%trans%}Use JS library{%endtrans%}</h3>
+	<p>{%trans%}A completely configurable viewer for your website{%endtrans%}</p>
 	<pre><code id="code-js-html">
 	</code></pre>
 	<script>
@@ -67,7 +67,7 @@
 			+'<div id="viewer" style="width: 500px; height: 300px"></div>\n\n'
 			+'&lt;script>\n'
 			+'\t// All options available are listed here\n'
-			+'\t// https://gitlab.com/geovisio/web-viewer/-/blob/develop/docs/02_Usage.md\n'
+			+'\t// https://gitlab.com/panoramax/clients/web-viewer/-/blob/develop/docs/02_Usage.md\n'
 			+'\tvar instance = new GeoVisio.default(\n'
 			+'\t\t"viewer",\n'
 			+'\t\t"'+baseUrl+'/api",\n'
@@ -79,15 +79,15 @@
 
 	<hr />
 
-	<h2>Links</h2>
+	<h2>{%trans%}Links{%endtrans%}</h2>
 	<p>
-		<a href="/viewer">Pictures viewer</a>
+		<a href="/viewer">{%trans%}Pictures viewer{%endtrans%}</a>
 		-
-		<a href="/api/docs/swagger">API docs</a>
+		<a href="/api/docs/swagger">{%trans%}API docs{%endtrans%}</a>
 		-
-		<a href="https://gitlab.com/geovisio/web-viewer/-/tree/develop/docs">JS library docs</a>
+		<a href="https://gitlab.com/panoramax/clients/web-viewer/-/tree/develop/docs">{%trans%}JS library docs{%endtrans%}</a>
 		-
-		<a href="https://gitlab.com/geovisio">Repositories</a>
+		<a href="https://gitlab.com/panoramax">{%trans%}Repositories{%endtrans%}</a>
 	</p>
 </body>
 </html>

geovisio/templates/viewer.html

@@ -6,7 +6,7 @@
 	<link rel="shortcut icon" href="/favicon.ico" />
 	<link rel="icon" type="image/svg+xml" href="/static/img/favicon.svg" />
 	<title>GeoVisio</title>
-	<link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/geovisio@~{{API_VERSION_MAJOR_MINOR}}/build/index.css" />
+	<link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/geovisio@~3.0/build/index.css" />
 	<style>
 		#viewer {
 			position: absolute;
@@ -19,10 +19,10 @@
 </head>
 <body>
 	<div id="viewer">
-		<noscript>You need to enable JavaScript to run this app.</noscript>
+		<noscript>{%trans%}You need to enable JavaScript to run this app.{%endtrans%}</noscript>
 	</div>
 
-	<script src="https://cdn.jsdelivr.net/npm/geovisio@~{{API_VERSION_MAJOR_MINOR}}/build/index.js"></script>
+	<script src="https://cdn.jsdelivr.net/npm/geovisio@~3.0/build/index.js"></script>
 	<script>
 		var instance = new GeoVisio.default(
 			"viewer",