geovisio 2.6.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

geovisio/web/map.py

@@ -1,12 +1,12 @@
 # Some parts of code here are heavily inspired from Paul Ramsey's work
 # See for reference : https://github.com/pramsey/minimal-mvt
 
-import psycopg
 import io
 from typing import Optional, Dict, Any, Tuple, List, Union
 from uuid import UUID
 from flask import Blueprint, current_app, send_file, request, jsonify, url_for
-from geovisio.utils import auth
+from flask_babel import gettext as _
+from geovisio.utils import auth, db
 from geovisio.utils.auth import Account
 from geovisio.web import params
 from geovisio.web.utils import user_dependant_response
@@ -21,15 +21,15 @@ ZOOM_PICTURES = 15
 
 def get_style_json(forUser: Optional[Union[UUID, str]] = None):
     # Get correct vector tiles URL
-    tilesUrl = url_for("map.getTile", x="111", y="222", z="333", format="mvt", _external=True)
+    tilesUrl = url_for("map.getTile", x="11111111", y="22222222", z="33333333", format="mvt", _external=True)
     sourceId = "geovisio"
     if forUser == "me":
-        tilesUrl = url_for("map.getMyTile", x="111", y="222", z="333", format="mvt", _external=True)
+        tilesUrl = url_for("map.getMyTile", x="11111111", y="22222222", z="33333333", format="mvt", _external=True)
         sourceId = "geovisio_me"
     elif forUser is not None:
-        tilesUrl = url_for("map.getUserTile", userId=forUser, x="111", y="222", z="333", format="mvt", _external=True)
+        tilesUrl = url_for("map.getUserTile", userId=forUser, x="11111111", y="22222222", z="33333333", format="mvt", _external=True)
         sourceId = f"geovisio_{str(forUser)}"
-    tilesUrl = tilesUrl.replace("111", "{x}").replace("222", "{y}").replace("333", "{z}")
+    tilesUrl = tilesUrl.replace("11111111", "{x}").replace("22222222", "{y}").replace("33333333", "{z}")
 
     # Display sequence on all zooms if user tiles, after grid on general tiles
     sequenceOpacity = ["interpolate", ["linear"], ["zoom"], ZOOM_GRID_SEQUENCES, 0, ZOOM_GRID_SEQUENCES + 1, 1] if forUser is None else 1
@@ -73,7 +73,7 @@ def get_style_json(forUser: Optional[Union[UUID, str]] = None):
                 "source": sourceId,
                 "source-layer": "grid",
                 "paint": {
-                    "fill-color": ["interpolate-hcl", ["linear"], ["get", "coef"], 0, "#FFCC80", 0.5, "#E65100", 1, "#BF360C"],
+                    "fill-color": ["interpolate", ["linear"], ["get", "coef"], 0, "#FFCC80", 0.5, "#E65100", 1, "#BF360C"],
                     "fill-opacity": [
                         "interpolate",
                         ["linear"],
@@ -120,32 +120,31 @@ def checkTileValidity(z, x, y, format):
     raises InvalidAPIUsage exceptions if parameters are not OK
     """
     if z is None or x is None or y is None or format is None:
-        raise errors.InvalidAPIUsage("One of required parameter is empty", status_code=404)
+        raise errors.InvalidAPIUsage(_("One of required parameter is empty"), status_code=404)
     if format not in ["pbf", "mvt"]:
-        raise errors.InvalidAPIUsage("Tile format is invalid, should be either pbf or mvt", status_code=400)
+        raise errors.InvalidAPIUsage(_("Tile format is invalid, should be either pbf or mvt"), status_code=400)
 
     size = 2**z
     if x >= size or y >= size:
-        raise errors.InvalidAPIUsage("X or Y parameter is out of bounds", status_code=404)
+        raise errors.InvalidAPIUsage(_("X or Y parameter is out of bounds"), status_code=404)
     if x < 0 or y < 0:
-        raise errors.InvalidAPIUsage("X or Y parameter is out of bounds", status_code=404)
+        raise errors.InvalidAPIUsage(_("X or Y parameter is out of bounds"), status_code=404)
     if z < 0 or z > 15:
-        raise errors.InvalidAPIUsage("Z parameter is out of bounds (should be 0-15)", status_code=404)
+        raise errors.InvalidAPIUsage(_("Z parameter is out of bounds (should be 0-15)"), status_code=404)
 
 
 def _getTile(z: int, x: int, y: int, format: str, onlyForUser: Optional[UUID] = None, filter: Optional[sql.SQL] = None):
     checkTileValidity(z, x, y, format)
 
-    with psycopg.connect(current_app.config["DB_URL"], options="-c statement_timeout=10000") as conn:
-        with conn.cursor() as cursor:
-            query, params = _get_query(z, x, y, onlyForUser, additional_filter=filter)
-            res = cursor.execute(query, params).fetchone()
+    query, params = _get_query(z, x, y, onlyForUser, additional_filter=filter)
 
-            if not res:
-                raise errors.InternalError("Impossible to get tile")
+    res = db.fetchone(current_app, query, params, timeout=10000)
 
-            res = res[0]
-            return send_file(io.BytesIO(res), mimetype="application/vnd.mapbox-vector-tile")
+    if not res:
+        raise errors.InternalError(_("Impossible to get tile"))
+
+    res = res[0]
+    return send_file(io.BytesIO(res), mimetype="application/vnd.mapbox-vector-tile")
 
 
 @bp.route("/map/style.json")
@@ -466,6 +465,7 @@ def getUserStyle(userId: UUID):
 
 
 @bp.route("/users/<uuid:userId>/map/<int:z>/<int:x>/<int:y>.<format>")
+@user_dependant_response(True)
 def getUserTile(userId: UUID, z: int, x: int, y: int, format: str):
     """Get pictures and sequences as vector tiles for a specific user.
     This tile will contain the same layers as the generic tiles (from `/map/z/x/y.format` route), but with sequences properties on all levels

geovisio/web/params.py

@@ -1,4 +1,5 @@
 from uuid import UUID
+
 from geovisio import errors
 import dateutil.parser
 from dateutil import tz
@@ -9,9 +10,12 @@ from werkzeug.datastructures import MultiDict
 from typing import Optional, Tuple, Any, List
 from pygeofilter.backends.sql import to_sql_where
 from pygeofilter.parsers.ecql import parse as ecql_parser
+from pygeofilter import ast
+from pygeofilter.backends.evaluator import Evaluator, handle
 from psycopg import sql
 from geovisio.utils.sequences import STAC_FIELD_MAPPINGS, STAC_FIELD_TO_SQL_FILTER
 from geovisio.utils.fields import SortBy, SQLDirection, SortByField
+from flask_babel import gettext as _
 
 
 RGX_SORTBY = re.compile("[+-]?[A-Za-z_].*(,[+-]?[A-Za-z_].*)*")
@@ -47,7 +51,7 @@ def parse_datetime(value, error, fallback_as_UTC=False):
     d = None
     try:
         d = datetime.datetime.fromisoformat(value)
-    except ValueError as e:
+    except ValueError:
         pass
     if not d:
         try:
@@ -81,17 +85,17 @@ def parse_datetime_interval(value: Optional[str]) -> Tuple[Optional[datetime.dat
     dates = value.split("/")
 
     if len(dates) == 1:
-        d = parse_datetime(dates[0], error=f"Invalid `datetime` argument", fallback_as_UTC=True)
+        d = parse_datetime(dates[0], error="Invalid `datetime` argument", fallback_as_UTC=True)
         return (d, d)
 
     elif len(dates) == 2:
         # Check if interval is closed or open-ended
         mind, maxd = dates
-        mind = None if mind == ".." else parse_datetime(mind, error=f"Invalid start date in `datetime` argument", fallback_as_UTC=True)
-        maxd = None if maxd == ".." else parse_datetime(maxd, error=f"Invalid end date in `datetime` argument", fallback_as_UTC=True)
+        mind = None if mind == ".." else parse_datetime(mind, error="Invalid start date in `datetime` argument", fallback_as_UTC=True)
+        maxd = None if maxd == ".." else parse_datetime(maxd, error="Invalid end date in `datetime` argument", fallback_as_UTC=True)
         return (mind, maxd)
     else:
-        raise errors.InvalidAPIUsage("Parameter datetime should contain one or two dates", status_code=400)
+        raise errors.InvalidAPIUsage(_("Parameter datetime should contain one or two dates"), status_code=400)
 
 
 def parse_bbox(value: Optional[Any], tryFallbacks=True):
@@ -161,12 +165,12 @@ def parse_bbox(value: Optional[Any], tryFallbacks=True):
                 or bbox[3] > 90
             ):
                 raise errors.InvalidAPIUsage(
-                    "Parameter bbox must contain valid longitude (-180 to 180) and latitude (-90 to 90) values", status_code=400
+                    _("Parameter bbox must contain valid longitude (-180 to 180) and latitude (-90 to 90) values"), status_code=400
                 )
             else:
                 return bbox
         except ValueError:
-            raise errors.InvalidAPIUsage("Parameter bbox must be in format [minX, minY, maxX, maxY]", status_code=400)
+            raise errors.InvalidAPIUsage(_("Parameter bbox must be in format [minX, minY, maxX, maxY]"), status_code=400)
     else:
         return None
 
@@ -203,11 +207,11 @@ def parse_lonlat(values: Optional[Any], paramName: Optional[str] = None) -> Opti
     entries = parse_list(values, paramName=paramName)
 
     if entries is None or len(entries) != 2:
-        raise errors.InvalidAPIUsage(f"Parameter {paramName or ''} must be coordinates in lat,lon format", status_code=400)
+        raise errors.InvalidAPIUsage(_("Parameter %(p)s must be coordinates in lat,lon format", p=paramName or ""), status_code=400)
 
     return [
-        as_longitude(entries[0], f"Longitude in parameter {paramName or ''} is not valid (should be between -180 and 180)"),
-        as_latitude(entries[1], f"Latitude in parameter {paramName or ''} is not valid (should be between -90 and 90)"),
+        as_longitude(entries[0], _("Longitude in parameter %(p)s is not valid (should be between -180 and 180)", p=paramName or "")),
+        as_latitude(entries[1], _("Latitude in parameter %(p)s is not valid (should be between -90 and 90)", p=paramName or "")),
     ]
 
 
@@ -230,18 +234,20 @@ def parse_distance_range(values: Optional[str], paramName: Optional[str] = None)
         dists = values.split("-")
         if len(dists) != 2:
             raise errors.InvalidAPIUsage(
-                f"Parameter {paramName or ''} is invalid (should be a distance range in meters like \"5-15\")", status_code=400
+                _('Parameter %(p)s is invalid (should be a distance range in meters like "5-15")', p={paramName or ""}), status_code=400
             )
         try:
             dists = [int(d) for d in dists]
             if dists[0] > dists[1]:
-                raise errors.InvalidAPIUsage(f"Parameter {paramName or ''} has a min value greater than its max value", status_code=400)
+                raise errors.InvalidAPIUsage(
+                    _("Parameter %(p)s has a min value greater than its max value", p=paramName or ""), status_code=400
+                )
             else:
                 return dists
 
         except ValueError:
             raise errors.InvalidAPIUsage(
-                f"Parameter {paramName or ''} is invalid (should be a distance range in meters like \"5-15\")", status_code=400
+                _('Parameter %(p)s is invalid (should be a distance range in meters like "5-15")', p={paramName or ""}), status_code=400
             )
     else:
         return None
@@ -308,7 +314,7 @@ def parse_list(value: Optional[Any], tryFallbacks: bool = True, paramName: Optio
             value = value.replace("[", "").replace("]", "")
             res = [n.strip() for n in value.split(",")]
         else:
-            raise errors.InvalidAPIUsage(f"Parameter {paramName or ''} must be a valid list", status_code=400)
+            raise errors.InvalidAPIUsage(_("Parameter %(p)s must be a valid list", p=paramName or ""), status_code=400)
 
         if len(res) == 0:
             return None
@@ -328,10 +334,10 @@ def parse_filter(value: Optional[str]) -> Optional[sql.SQL]:
     SQL("(s.updated_at >= '2023-12-31')")
     >>> parse_filter("updated >= '2023-12-31' AND created < '2023-10-31'")
     SQL("((s.updated_at >= '2023-12-31') AND (s.inserted_at < '2023-10-31'))")
-    >>> parse_filter("status IN ('deleted','ready')")
-    SQL("s.status IN ('deleted', 'ready')")
+    >>> parse_filter("status IN ('deleted','ready')") # when we ask for deleted, we should also have hidden collections
+    SQL("s.status IN ('deleted', 'ready', 'hidden')")
     >>> parse_filter("status = 'deleted' OR status = 'ready'")
-    SQL("((s.status = 'deleted') OR (s.status = 'ready'))")
+    SQL("(((s.status = 'deleted') OR (s.status = 'hidden')) OR (s.status = 'ready'))")
     >>> parse_filter('invalid = 10') # doctest: +IGNORE_EXCEPTION_DETAIL
     Traceback (most recent call last):
     geovisio.errors.InvalidAPIUsage: Unsupported filter parameter
@@ -342,14 +348,51 @@ def parse_filter(value: Optional[str]) -> Optional[sql.SQL]:
     if value is not None and len(value) > 0:
         try:
             filterAst = ecql_parser(value)
-            f = to_sql_where(filterAst, STAC_FIELD_TO_SQL_FILTER).replace('"', "")
-            return sql.SQL(f)
+            altered_ast = _alterFilterAst(filterAst)  # type: ignore
+
+            f = to_sql_where(altered_ast, STAC_FIELD_TO_SQL_FILTER).replace('"', "")
+            return sql.SQL(f)  # type: ignore
         except:
-            raise errors.InvalidAPIUsage(f"Unsupported filter parameter", status_code=400)
+            raise errors.InvalidAPIUsage(_("Unsupported filter parameter"), status_code=400)
     else:
         return None
 
 
+class _FilterAstUpdated(Evaluator):
+    """
+    We alter the parsed AST in order to always query for 'hidden' pictures when we query for 'deleted' ones
+
+    The rational here is that for non-owned pictures/sequences, when a pictures/sequence is 'hidden' it should be advertised as 'deleted'.
+
+    This is especially important for crawler like the meta-catalog, since they should also delete the sequence/picture when it is hidden
+    """
+
+    @handle(ast.Equal)
+    def eq(self, node, lhs, rhs):
+        if lhs == ast.Attribute("status") and rhs == "deleted":
+            return ast.Or(node, ast.Equal(ast.Attribute("status"), "hidden"))  # type: ignore
+        return node
+
+    @handle(ast.Or)
+    def or_(self, node, lhs, rhs):
+        return ast.Or(lhs, rhs)
+
+    @handle(ast.In)
+    def in_(self, node, lhs, *options):
+        if "deleted" in node.sub_nodes:
+            node.sub_nodes.append("hidden")
+        return node
+
+    def adopt(self, node, *sub_args):
+        return node
+
+
+def _alterFilterAst(ast: ast.Node):
+    filtered = _FilterAstUpdated().evaluate(ast)
+
+    return filtered
+
+
 def parse_sortby(value: Optional[str]) -> Optional[SortBy]:
     """Reads STAC/OGC sortby parameter, and sends a SQL ORDER BY string.
 
@@ -395,7 +438,7 @@ def parse_sortby(value: Optional[str]) -> Optional[SortBy]:
 
                 # Check if in value mapping
                 if vOnly not in STAC_FIELD_MAPPINGS:
-                    raise errors.InvalidAPIUsage(f"Unsupported sortby parameter: invalid column name", status_code=400)
+                    raise errors.InvalidAPIUsage(_("Unsupported sortby parameter: invalid column name"), status_code=400)
                 field_mapping = STAC_FIELD_MAPPINGS[vOnly]
 
                 orders.append(SortByField(field=field_mapping, direction=direction))
@@ -403,7 +446,7 @@ def parse_sortby(value: Optional[str]) -> Optional[SortBy]:
             # Create definitive ORDER string
             return SortBy(fields=orders)
         else:
-            raise errors.InvalidAPIUsage(f"Unsupported sortby parameter: syntax isn't correct", status_code=400)
+            raise errors.InvalidAPIUsage(_("Unsupported sortby parameter: syntax isn't correct"), status_code=400)
     else:
         return None
 
@@ -429,10 +472,10 @@ def parse_collections_limit(limit: Optional[str]) -> int:
     try:
         int_limit = int(limit)
     except ValueError:
-        raise errors.InvalidAPIUsage(f"limit parameter should be a valid, positive integer (between 1 and {SEQUENCES_MAX_FETCH})")
+        raise errors.InvalidAPIUsage(_("limit parameter should be a valid, positive integer (between 1 and %(v)s)", v=SEQUENCES_MAX_FETCH))
 
     if int_limit < 1 or int_limit > SEQUENCES_MAX_FETCH:
-        raise errors.InvalidAPIUsage(f"limit parameter should be an integer between 1 and {SEQUENCES_MAX_FETCH}")
+        raise errors.InvalidAPIUsage(_("limit parameter should be an integer between 1 and %(v)s", v=SEQUENCES_MAX_FETCH))
     else:
         return int_limit
 
@@ -443,7 +486,7 @@ def as_longitude(value: str, error):
     except ValueError as e:
         raise errors.InvalidAPIUsage(message=error, payload={"details": {"error": str(e)}})
     if l < -180 or l > 180:
-        raise errors.InvalidAPIUsage(message=error, payload={"details": {"error": "longitude needs to be between -180 and 180"}})
+        raise errors.InvalidAPIUsage(message=error, payload={"details": {"error": _("longitude needs to be between -180 and 180")}})
     return l
 
 
@@ -453,7 +496,7 @@ def as_latitude(value: str, error):
     except ValueError as e:
         raise errors.InvalidAPIUsage(message=error, payload={"details": {"error": str(e)}})
     if l < -90 or l > 90:
-        raise errors.InvalidAPIUsage(message=error, payload={"details": {"error": "latitude needs to be between -90 and 90"}})
+        raise errors.InvalidAPIUsage(message=error, payload={"details": {"error": _("latitude needs to be between -90 and 90")}})
     return l
 
 

geovisio/web/pictures.py

@@ -1,8 +1,7 @@
-import os
-from itertools import repeat
-from flask import Blueprint, current_app, request
+from flask import Blueprint, current_app
 from geovisio import utils, errors
 from flask import redirect
+from flask_babel import gettext as _
 import logging
 
 bp = Blueprint("pictures", __name__, url_prefix="/api/pictures")
@@ -25,7 +24,7 @@ def getPictureHD(pictureId, format):
             type: string
         - name: format
           in: path
-          description: Wanted format for output image (either jpg or webp)
+          description: Wanted format for output image (for the moment only jpg)
           required: true
           schema:
             type: string
@@ -37,10 +36,6 @@ def getPictureHD(pictureId, format):
                     schema:
                         type: string
                         format: binary
-                image/webp:
-                    schema:
-                        type: string
-                        format: binary
     """
 
     utils.pictures.checkFormatParam(format)
@@ -55,7 +50,7 @@ def getPictureHD(pictureId, format):
     try:
         picture = fses.permanent.openbin(utils.pictures.getHDPicturePath(pictureId))
     except:
-        raise errors.InvalidAPIUsage("Unable to read picture on filesystem", status_code=500)
+        raise errors.InvalidAPIUsage(_("Unable to read picture on filesystem"), status_code=500)
 
     return utils.pictures.sendInFormat(picture, "jpeg", format)
 
@@ -75,7 +70,7 @@ def getPictureSD(pictureId, format):
             type: string
         - name: format
           in: path
-          description: Wanted format for output image (either jpg or webp)
+          description: Wanted format for output image (for the moment only jpg)
           required: true
           schema:
             type: string
@@ -87,10 +82,6 @@ def getPictureSD(pictureId, format):
                     schema:
                         type: string
                         format: binary
-                image/webp:
-                    schema:
-                        type: string
-                        format: binary
     """
     utils.pictures.checkFormatParam(format)
 
@@ -104,7 +95,7 @@ def getPictureSD(pictureId, format):
     try:
         picture = fses.derivates.openbin(utils.pictures.getPictureFolderPath(pictureId) + "/sd.jpg")
     except:
-        raise errors.InvalidAPIUsage("Unable to read picture on filesystem", status_code=500)
+        raise errors.InvalidAPIUsage(_("Unable to read picture on filesystem"), status_code=500)
 
     return utils.pictures.sendInFormat(picture, "jpeg", format)
 
@@ -124,7 +115,7 @@ def getPictureThumb(pictureId, format):
             type: string
         - name: format
           in: path
-          description: Wanted format for output image (either jpg or webp)
+          description: Wanted format for output image (for the moment only jpg)
           required: true
           schema:
             type: string
@@ -136,10 +127,6 @@ def getPictureThumb(pictureId, format):
                     schema:
                         type: string
                         format: binary
-                image/webp:
-                    schema:
-                        type: string
-                        format: binary
     """
     return utils.pictures.sendThumbnail(pictureId, format)
 
@@ -171,7 +158,7 @@ def getPictureTile(pictureId, col, row, format):
             type: number
         - name: format
           in: path
-          description: Wanted format for output image (either jpg or webp)
+          description: Wanted format for output image (for the moment only jpg)
           required: true
           schema:
             type: string
@@ -183,10 +170,6 @@ def getPictureTile(pictureId, col, row, format):
                     schema:
                         type: string
                         format: binary
-                image/webp:
-                    schema:
-                        type: string
-                        format: binary
     """
 
     utils.pictures.checkFormatParam(format)
@@ -201,27 +184,27 @@ def getPictureTile(pictureId, col, row, format):
     picPath = f"{utils.pictures.getPictureFolderPath(pictureId)}/tiles/{col}_{row}.jpg"
 
     if metadata["type"] == "flat":
-        raise errors.InvalidAPIUsage("Tiles are not available for flat pictures", status_code=404)
+        raise errors.InvalidAPIUsage(_("Tiles are not available for flat pictures"), status_code=404)
 
     try:
         col = int(col)
     except:
-        raise errors.InvalidAPIUsage("Column parameter is invalid, should be an integer", status_code=404)
+        raise errors.InvalidAPIUsage(_("Column parameter is invalid, should be an integer"), status_code=404)
 
     if col < 0 or col >= metadata["cols"]:
-        raise errors.InvalidAPIUsage("Column parameter is invalid", status_code=404)
+        raise errors.InvalidAPIUsage(_("Column parameter is invalid"), status_code=404)
 
     try:
         row = int(row)
     except:
-        raise errors.InvalidAPIUsage("Row parameter is invalid, should be an integer", status_code=404)
+        raise errors.InvalidAPIUsage(_("Row parameter is invalid, should be an integer"), status_code=404)
 
     if row < 0 or row >= metadata["rows"]:
-        raise errors.InvalidAPIUsage("Row parameter is invalid", status_code=404)
+        raise errors.InvalidAPIUsage(_("Row parameter is invalid"), status_code=404)
 
     try:
         picture = fses.derivates.openbin(picPath)
     except:
-        raise errors.InvalidAPIUsage("Unable to read picture on filesystem", status_code=500)
+        raise errors.InvalidAPIUsage(_("Unable to read picture on filesystem"), status_code=500)
 
     return utils.pictures.sendInFormat(picture, "jpeg", format)