geovisio 2.7.0__py3-none-any.whl → 2.8.0__py3-none-any.whl

geovisio/web/collections.py

@@ -1,6 +1,9 @@
 from enum import Enum
+from attr import dataclass
 from geovisio import errors, utils, db
 from geovisio.utils import auth, sequences
+from geovisio.utils.params import validation_error
+from geovisio.utils.semantics import SemanticTagUpdate, Entity, EntityType, update_tags
 from geovisio.web.params import (
     parse_datetime,
     parse_datetime_interval,
@@ -18,7 +21,8 @@ from geovisio.utils.fields import SortBy, SortByField, SQLDirection, Bounds, BBo
 from geovisio.web.rss import dbSequencesToGeoRSS
 from psycopg.rows import dict_row
 from psycopg.sql import SQL
-from flask import current_app, request, url_for, Blueprint
+from pydantic import BaseModel, Field, ValidationError, field_validator
+from flask import current_app, request, url_for, Blueprint, stream_with_context
 from flask_babel import gettext as _
 from geovisio.web.utils import (
     STAC_VERSION,
@@ -30,7 +34,7 @@ from geovisio.web.utils import (
     get_root_link,
     removeNoneInDict,
 )
-from typing import Optional
+from typing import List, Optional
 
 
 bp = Blueprint("stac_collections", __name__, url_prefix="/api")
@@ -82,11 +86,15 @@ def dbSequenceToStacCollection(dbSeq, description="A sequence of geolocated pict
         {
             "type": "Collection",
             "stac_version": STAC_VERSION,
-            "stac_extensions": ["https://stac-extensions.github.io/stats/v0.2.0/schema.json"],  # For stats: fields
+            "stac_extensions": [
+                "https://stac-extensions.github.io/stats/v0.2.0/schema.json",  # For stats: fields
+                "https://stac.linz.govt.nz/v0.0.15/quality/schema.json",  # For quality: fields
+            ],
             "id": str(dbSeq["id"]),
             "title": str(dbSeq["name"]),
             "description": description,
             "keywords": ["pictures", str(dbSeq["name"])],
+            "semantics": dbSeq["semantics"] if "semantics" in dbSeq else None,
             "license": current_app.config["API_PICTURES_LICENSE_SPDX_ID"],
             "created": dbTsToStac(dbSeq["created"]),
             "updated": dbTsToStac(dbSeq.get("updated")),
@@ -96,6 +104,10 @@ def dbSequenceToStacCollection(dbSeq, description="A sequence of geolocated pict
             "geovisio:sorted-by": dbSeq.get("current_sort"),
             "geovisio:upload-software": userAgentToClient(dbSeq.get("user_agent")).value,
             "geovisio:length_km": dbSeq.get("length_km"),
+            "quality:horizontal_accuracy": (
+                float("{:.1f}".format(dbSeq["computed_gps_accuracy"])) if dbSeq.get("computed_gps_accuracy") else None
+            ),
+            "quality:horizontal_accuracy_type": "95% confidence interval" if "computed_gps_accuracy" in dbSeq else None,
             "providers": [
                 {"name": dbSeq["account_name"], "roles": ["producer"], "id": str(dbSeq["account_id"])},
             ],
@@ -110,7 +122,14 @@ def dbSequenceToStacCollection(dbSeq, description="A sequence of geolocated pict
                     ]
                 },
             },
-            "summaries": cleanNoneInDict({"pers:interior_orientation": dbSeq.get("metas")}),
+            "summaries": cleanNoneInDict(
+                {
+                    "pers:interior_orientation": dbSeq.get("metas"),
+                    "panoramax:horizontal_pixel_density": (
+                        [dbSeq["computed_h_pixel_density"]] if "computed_h_pixel_density" in dbSeq else None
+                    ),
+                }
+            ),
             "stats:items": removeNoneInDict({"count": nb_pic}),
             "links": cleanNoneInList(
                 [
@@ -374,8 +393,19 @@ def getCollection(collectionId):
                 max_picture_ts AS maxts,
                 nb_pictures AS nbpic,
                 s.user_agent,
-                ROUND(ST_Length(s.geom::geography)) / 1000 as length_km
+                ROUND(ST_Length(s.geom::geography)) / 1000 as length_km,
+                s.computed_h_pixel_density,
+                s.computed_gps_accuracy,
+                t.semantics
             FROM sequences s
+            LEFT JOIN (
+                SELECT sequence_id, json_agg(json_strip_nulls(json_build_object(
+                    'key', key,
+                    'value', value
+                ))) AS semantics
+                FROM sequences_semantics
+                GROUP BY sequence_id
+            ) t ON t.sequence_id = s.id
             JOIN accounts ON s.account_id = accounts.id, (
                 SELECT
                     array_agg(DISTINCT jsonb_build_object(
@@ -523,13 +553,83 @@ def postCollection(account=None):
     )
 
 
+class PatchCollectionParameter(BaseModel):
+    """Parameters used to add an item to an UploadSet"""
+
+    relative_heading: Optional[int] = None
+    """The relative heading (in degrees), offset based on movement path (0° = looking forward, -90° = looking left, 90° = looking right). Headings are unchanged if this parameter is not set."""
+    visible: Optional[bool] = None
+    """Should the sequence be publicly visible ?"""
+    title: Optional[str] = Field(max_length=250, default=None)
+    """The sequence title (publicly displayed)"""
+    sortby: Optional[str] = None
+    """Define the pictures sort order based on given property. Sort order is defined based on preceding '+' (asc) or '-' (desc).
+
+Available properties are:
+* `gpsdate`: sort by GPS datetime
+* `filedate`: sort by the camera-generated capture date. This is based on EXIF tags `Exif.Image.DateTimeOriginal`, `Exif.Photo.DateTimeOriginal`, `Exif.Image.DateTime` or `Xmp.GPano.SourceImageCreateTime` (in this order).
+* `filename`: sort by the original picture file name
+
+If unset, sort order is unchanged."""
+    semantics: Optional[List[SemanticTagUpdate]] = None
+    """Tags to update on the picture. By default each tag will be added to the picture's tags, but you can change this behavior by setting the `action` parameter to `delete`.
+    
+    Like:
+[
+    {"key": "some_key", "value": "some_value", "action": "delete"},
+    {"key": "some_key", "value": "some_new_value"}
+]
+
+    Note that updating tags is only possible with JSON data, not with form-data."""
+
+    def has_override(self) -> bool:
+        return self.model_fields_set
+
+    @field_validator("visible", mode="before")
+    @classmethod
+    def parse_visible(cls, value):
+        if value not in ["true", "false"]:
+            raise errors.InvalidAPIUsage(_("Picture visibility parameter (visible) should be either unset, true or false"), status_code=400)
+        return value == "true"
+
+    @field_validator("sortby", mode="before")
+    @classmethod
+    def check_sortby(cls, value):
+        if value not in ["+gpsdate", "-gpsdate", "+filedate", "-filedate", "+filename", "-filename"]:
+            raise errors.InvalidAPIUsage(_("Sort order parameter is invalid"), status_code=400)
+        return value
+
+    @field_validator("relative_heading", mode="before")
+    @classmethod
+    def parse_relative_heading(cls, value):
+        try:
+            relHeading = int(value)
+            if relHeading < -180 or relHeading > 180:
+                raise ValueError()
+            return relHeading
+        except ValueError:
+            raise errors.InvalidAPIUsage(
+                _("Relative heading is not valid, should be an integer in degrees from -180 to 180"), status_code=400
+            )
+
+    def has_only_semantics_updates(self):
+        return self.model_fields_set == {"semantics"}
+
+
 @bp.route("/collections/<uuid:collectionId>", methods=["PATCH"])
 @auth.login_required()
 def patchCollection(collectionId, account):
     """Edits properties of an existing collection
+
+    Note that there are rules on the editing of a sequence's metadata:
+
+    - Only the owner of a picture can change its visibility and title
+    - For core metadata (relative_heading, sort_by), the owner can restrict their change by other accounts (see `collaborative_metadata` field in `/api/users/me`) and if not explicitly defined by the user, the instance's default value is used.
+    - Everyone can add/edit/delete semantics tags.
     ---
     tags:
         - Editing
+        - Tags
     parameters:
         - name: collectionId
           in: path
@@ -561,48 +661,18 @@ def patchCollection(collectionId, account):
     """
 
     # Parse received parameters
-    metadata = {}
     content_type = (request.headers.get("Content-Type") or "").split(";")[0]
-    for param in ["visible", "title", "relative_heading", "sortby"]:
+    metadata = None
+    try:
         if request.is_json and request.json:
-            metadata[param] = request.json.get(param)
+            metadata = PatchCollectionParameter(**request.json)
         elif content_type in ["multipart/form-data", "application/x-www-form-urlencoded"]:
-            metadata[param] = request.form.get(param)
-
-    # Check if visibility param is valid
-    visible = metadata.get("visible")
-    if visible is not None:
-        if visible in ["true", "false"]:
-            visible = visible == "true"
-        else:
-            raise errors.InvalidAPIUsage(_("Picture visibility parameter (visible) should be either unset, true or false"), status_code=400)
-
-    # Check if title is valid
-    newTitle = metadata.get("title")
-    if newTitle is not None:
-        if not (isinstance(newTitle, str) and len(newTitle) <= 250):
-            raise errors.InvalidAPIUsage(_("Sequence title is not valid, should be a string with a max of 250 characters"), status_code=400)
-
-    # Check if sortby is valid
-    sortby = metadata.get("sortby")
-    if sortby is not None:
-        if sortby not in ["+gpsdate", "-gpsdate", "+filedate", "-filedate", "+filename", "-filename"]:
-            raise errors.InvalidAPIUsage(_("Sort order parameter is invalid"), status_code=400)
-
-    # Check if relative_heading is valid
-    relHeading = metadata.get("relative_heading")
-    if relHeading is not None:
-        try:
-            relHeading = int(relHeading)
-            if relHeading < -180 or relHeading > 180:
-                raise ValueError()
-        except ValueError:
-            raise errors.InvalidAPIUsage(
-                _("Relative heading is not valid, should be an integer in degrees from -180 to 180"), status_code=400
-            )
+            metadata = PatchCollectionParameter(**request.form)
+    except ValidationError as ve:
+        raise errors.InvalidAPIUsage(_("Impossible to parse parameters"), payload=validation_error(ve))
 
     # If no parameter is changed, no need to contact DB, just return sequence as is
-    if {visible, newTitle, relHeading, sortby} == {None}:
+    if metadata is None or not metadata.has_override():
         return getCollection(collectionId)
 
     # Check if sequence exists and if given account is authorized to edit
@@ -617,9 +687,24 @@ def patchCollection(collectionId, account):
                 if not seq:
                     raise errors.InvalidAPIUsage(_("Collection %(c)s wasn't found in database", c=collectionId), status_code=404)
 
-                # Account associated to sequence doesn't match current user
                 if account is not None and account.id != str(seq["account_id"]):
-                    raise errors.InvalidAPIUsage(_("You're not authorized to edit this sequence"), status_code=403)
+                    # Only owner of the sequence is allower to change its visibility and title
+                    # tags and headings can be changed by anyone
+                    if metadata.visible is not None or metadata.title is not None:
+                        raise errors.InvalidAPIUsage(
+                            _(
+                                "You're not authorized to edit those fields for this sequence. Only the owner can change the visibility and the title"
+                            ),
+                            status_code=403,
+                        )
+
+                    # for core metadata editing (all appart the semantic tags), we check if the user has allowed it
+                    if not metadata.has_only_semantics_updates():
+                        if not auth.account_allow_collaborative_editing(seq["account_id"]):
+                            raise errors.InvalidAPIUsage(
+                                _("You're not authorized to edit this sequence, collaborative editing is not allowed"),
+                                status_code=403,
+                            )
 
                 oldStatus = seq["status"]
                 oldMetadata = seq["metadata"]
@@ -627,25 +712,26 @@ def patchCollection(collectionId, account):
 
                 # Check if sequence is in a preparing/broken/... state so no edit possible
                 if oldStatus not in ["ready", "hidden"]:
-                    raise errors.InvalidAPIUsage(
-                        _("Sequence %(c)s is in %(s)s state, its visibility can't be changed for now", c=collectionId, s=oldStatus),
-                        status_code=400,
-                    )
+                    if metadata.visible is not None:
+                        raise errors.InvalidAPIUsage(
+                            _("Sequence %(c)s is in %(s)s state, its visibility can't be changed for now", c=collectionId, s=oldStatus),
+                            status_code=400,
+                        )
 
                 sqlUpdates = []
                 sqlParams = {"id": collectionId, "account": account.id}
 
-                if visible is not None:
-                    newStatus = "ready" if visible is True else "hidden"
+                if metadata.visible is not None:
+                    newStatus = "ready" if metadata.visible is True else "hidden"
                     if newStatus != oldStatus:
                         sqlUpdates.append(SQL("status = %(status)s"))
                         sqlParams["status"] = newStatus
 
                 new_metadata = {}
-                if newTitle is not None and oldTitle != newTitle:
-                    new_metadata["title"] = newTitle
-                if relHeading:
-                    new_metadata["relative_heading"] = relHeading
+                if metadata.title is not None and oldTitle != metadata.title:
+                    new_metadata["title"] = metadata.title
+                if metadata.relative_heading:
+                    new_metadata["relative_heading"] = metadata.relative_heading
 
                 if new_metadata:
                     sqlUpdates.append(SQL("metadata = metadata || %(new_metadata)s"))
@@ -653,9 +739,9 @@ def patchCollection(collectionId, account):
 
                     sqlParams["new_metadata"] = Jsonb(new_metadata)
 
-                if sortby is not None:
+                if metadata.sortby is not None:
                     sqlUpdates.append(SQL("current_sort = %(sort)s"))
-                    sqlParams["sort"] = sortby
+                    sqlParams["sort"] = metadata.sortby
 
                 if len(sqlUpdates) > 0:
                     # Note: we set the field `last_account_to_edit` to track who changed the collection last (later we'll make it possible for everybody to edit some collection fields)
@@ -674,25 +760,29 @@ def patchCollection(collectionId, account):
                     )
 
                 # Edits picture sort order
-                if sortby is not None:
-                    direction = sequences.Direction(sortby[0])
-                    order = sequences.CollectionSortOrder(sortby[1:])
+                if metadata.sortby is not None:
+                    direction = sequences.Direction(metadata.sortby[0])
+                    order = sequences.CollectionSortOrder(metadata.sortby[1:])
                     sequences.sort_collection(cursor, collectionId, sequences.CollectionSort(order=order, direction=direction))
-                    if not relHeading:
+                    if not metadata.relative_heading:
                         # if we do not plan to override headings specifically, we recompute headings that have not bee provided by the users
                         # with the new movement track
                         sequences.update_headings(cursor, collectionId, editingAccount=account.id)
 
                 # Edits relative heading of pictures in sequence
-                if relHeading is not None:
+                if metadata.relative_heading is not None:
                     # New heading is computed based on sequence movement track
                     #   We take each picture and its following, compute azimuth,
                     #   then add given relative heading to offset picture heading.
                     #   Last picture is computed based on previous one in sequence.
                     sequences.update_headings(
-                        cursor, collectionId, relativeHeading=relHeading, updateOnlyMissing=False, editingAccount=account.id
+                        cursor, collectionId, relativeHeading=metadata.relative_heading, updateOnlyMissing=False, editingAccount=account.id
                     )
 
+                if metadata.semantics is not None:
+                    # semantic tags are managed separately
+                    update_tags(cursor, Entity(type=EntityType.seq, id=collectionId), metadata.semantics, account=account.id)
+
         # Redirect response to a classic GET
         return getCollection(collectionId)
 
@@ -825,6 +915,60 @@ FROM items i;""",
         return {"status": sequence_status["status"], "items": pics}
 
 
+def send_collections_as_csv(collection_request: CollectionsRequest):
+    """Retrieves all collections of a given user as a CSV file.
+
+    The response is streamed from the database to be more efficient, so for the moment we do not support many parameters
+    """
+    if collection_request.pagination_filter:
+        raise errors.InvalidAPIUsage(_("CSV export does not support pagination"), status_code=400)
+    if collection_request.filters():
+        raise errors.InvalidAPIUsage(_("CSV export does not support filters"), status_code=400)
+    if collection_request.sort_by != SortBy(fields=[SortByField(field=STAC_FIELD_MAPPINGS["created"], direction=SQLDirection.DESC)]):
+        raise errors.InvalidAPIUsage(_("CSV export does not support sorting by anything but creation date"), status_code=400)
+
+    def generate_csv():
+        # yield f"{','.join([f.name for f in CSV_FIELDS])}\n"
+        filters = [SQL("account_id = %(account)s")]
+        params = {"account": collection_request.user_id}
+        filters.append(SQL("status != 'deleted'") if collection_request.userOwnsAllCollections else SQL("status = 'ready'"))
+
+        with db.cursor(current_app) as cursor:
+
+            with cursor.copy(
+                SQL(
+                    """COPY (
+SELECT 
+    s.id AS id,
+    s.status AS status,
+    s.metadata->>'title' AS name,
+    s.inserted_at AS created,
+    s.updated_at AS updated,
+    s.computed_capture_date AS capture_date,
+    s.min_picture_ts AS minimum_capture_time,
+    s.max_picture_ts AS maximum_capture_time,
+    ST_XMin(s.bbox) AS min_x,
+    ST_YMin(s.bbox) AS min_y,
+    ST_XMax(s.bbox) AS max_x,
+    ST_YMax(s.bbox) AS max_y,
+    s.nb_pictures AS nb_pictures,
+    ROUND(ST_Length(s.geom::geography)) / 1000 AS length_km,
+    s.computed_h_pixel_density AS computed_h_pixel_density,
+    s.computed_gps_accuracy AS computed_gps_accuracy
+FROM sequences s 
+WHERE {filter}
+ORDER BY s.inserted_at DESC
+) TO STDOUT CSV HEADER"""
+                ).format(filter=SQL(" AND ").join(filters)),
+                params,
+            ) as copy:
+
+                for a in copy:
+                    yield bytes(a)
+
+    return stream_with_context(generate_csv()), {"Content-Disposition": "attachment"}
+
+
 @bp.route("/users/<uuid:userId>/collection")
 @auth.isUserIdMatchingCurrentAccount()
 def getUserCollection(userId, userIdMatchesAccount=False):
@@ -838,6 +982,8 @@ def getUserCollection(userId, userIdMatchesAccount=False):
 
     Note that on paginated results, filter can only be used with column used in sortby parameter.
 
+    The result can also be a CSV file, if the "Accept" header is set to "text/csv", or if the "format" query parameter is set to "csv".
+    Note that when requesting a CSV file, the filters/sortby/pagination parameters are not supported, and `limit` is ignored, you always get the full list of collections.
     ---
     tags:
         - Sequences
@@ -849,6 +995,14 @@ def getUserCollection(userId, userIdMatchesAccount=False):
           required: true
           schema:
             type: string
+        - name: format
+          in: query
+          description: Expected output format (STAC JSON or a csv file). Note that the CSV format support less parameters than the JSON format (cf documentation).
+          required: false
+          schema:
+            type: string
+            enum: [csv, json]
+            default: json
         - $ref: '#/components/parameters/STAC_collections_limit'
         - $ref: '#/components/parameters/STAC_collections_filter'
         - $ref: '#/components/parameters/STAC_bbox'
@@ -860,8 +1014,20 @@ def getUserCollection(userId, userIdMatchesAccount=False):
                 application/json:
                     schema:
                         $ref: '#/components/schemas/GeoVisioCollectionOfCollection'
+
+                text/csv:
+                    schema:
+                        $ref: '#/components/schemas/GeoVisioCSVCollections'
     """
 
+    # Expected output format
+    format = request.args["format"] if request.args.get("format") in ["csv", "json"] else "json"
+    if (
+        request.args.get("format") is None
+        and request.accept_mimetypes.best_match(["application/json", "text/csv"], "application/json") == "text/csv"
+    ):
+        format = "csv"
+
     # Sort-by parameter
     sortBy = parse_sortby(request.args.get("sortby"))
     if not sortBy:
@@ -876,7 +1042,9 @@ def getUserCollection(userId, userIdMatchesAccount=False):
     collection_request.pagination_filter = parse_filter(request.args.get("page"))
 
     # Limit parameter
-    collection_request.limit = parse_collections_limit(request.args.get("limit"))
+    # if not specified, the default with CSV it 1000. if there are more, the paginated API should be used
+    arg_limit = request.args.get("limit")
+    collection_request.limit = parse_collections_limit(arg_limit)
     collection_request.user_id = userId
 
     # Bounding box
@@ -911,6 +1079,9 @@ def getUserCollection(userId, userIdMatchesAccount=False):
             raise errors.InvalidAPIUsage(_("Impossible to find user %(u)s", u=userId))
         userName = userName["name"]
 
+        if format == "csv":
+            return send_collections_as_csv(collection_request)
+
         meta_collection = cursor.execute(
             SQL(
                 """SELECT
@@ -984,6 +1155,7 @@ def getUserCollection(userId, userIdMatchesAccount=False):
     )
     collection = dbSequenceToStacCollection(meta_collection, description=f"List of all sequences of user {userName}")
 
+    collection["stats:collections"] = removeNoneInDict({"count": meta_collection["nbseq"]})
     additional_filters = None
     if collection_request.user_filter is not None:
         # if some filters were given, we continue to pass them to the pagination

geovisio/web/configuration.py

@@ -28,11 +28,14 @@ def configuration():
         {
             "name": _get_translated(apiSum.name, userLang),
             "description": _get_translated(apiSum.description, userLang),
+            "geo_coverage": _get_translated(apiSum.geo_coverage, userLang),
             "logo": apiSum.logo,
             "color": str(apiSum.color),
+            "email": apiSum.email,
             "auth": _auth_configuration(),
             "license": _license_configuration(),
             "version": get_api_version(),
+            "pages": _get_pages(),
         }
     )
 
@@ -47,7 +50,11 @@ def _auth_configuration():
     if auth.oauth_provider is None:
         return {"enabled": False}
     else:
-        return {"enabled": True, "user_profile": {"url": auth.oauth_provider.user_profile_page_url()}}
+        return {
+            "enabled": True,
+            "user_profile": {"url": auth.oauth_provider.user_profile_page_url()},
+            "enforce_tos_acceptance": flask.current_app.config["API_ENFORCE_TOS_ACCEPTANCE"],
+        }
 
 
 def _license_configuration():
@@ -56,3 +63,12 @@ def _license_configuration():
     if u:
         l["url"] = u
     return l
+
+
+def _get_pages():
+    from geovisio.utils import db
+    from flask import current_app
+
+    pages = db.fetchall(current_app, "SELECT distinct(name) FROM pages")
+
+    return [p[0] for p in pages]