geovisio 2.8.1__py3-none-any.whl → 2.10.0__py3-none-any.whl

geovisio/utils/sequences.py

@@ -1,5 +1,8 @@
+from operator import ne
+from click import Option
+from numpy import sort
 import psycopg
-from flask import current_app, url_for
+from flask import current_app, g, url_for
 from flask_babel import gettext as _
 from psycopg.types.json import Jsonb
 from psycopg.sql import SQL, Composable
@@ -11,7 +14,7 @@ from uuid import UUID
 from enum import Enum
 from geovisio.utils import db
 from geovisio.utils.auth import Account
-from geovisio.utils.fields import FieldMapping, SortBy, SQLDirection, BBox, Bounds
+from geovisio.utils.fields import FieldMapping, SortBy, SQLDirection, BBox, Bounds, SortByField
 from geopic_tag_reader import reader
 from pathlib import PurePath
 from geovisio import errors, utils
@@ -39,6 +42,7 @@ STAC_FIELD_MAPPINGS = {
         FieldMapping(sql_column=SQL("updated_at"), stac="updated"),
         FieldMapping(sql_column=SQL("computed_capture_date"), stac="datetime"),
         FieldMapping(sql_column=SQL("status"), stac="status"),
+        FieldMapping(sql_column=SQL("id"), stac="id"),
     ]
 }
 STAC_FIELD_TO_SQL_FILTER = {p.stac: p.sql_filter.as_string(None) for p in STAC_FIELD_MAPPINGS.values()}
@@ -51,8 +55,8 @@ class Collections:
     """
 
     collections: List[Dict[Any, Any]] = field(default_factory=lambda: [])
-    # Bounds of the field used by the first field of the `ORDER BY` (usefull especially for pagination)
-    query_first_order_bounds: Optional[Bounds] = None
+    # Bounds of the field used by the first field of the `ORDER BY` (useful especially for pagination)
+    query_bounds: Optional[Bounds] = None
 
 
 @dataclass
@@ -153,23 +157,24 @@ def get_collections(request: CollectionsRequest) -> Collections:
                 s.min_picture_ts AS mints,
                 s.max_picture_ts AS maxts,
                 s.nb_pictures AS nbpic,
+                s.upload_set_id,
                 {status},
                 s.computed_capture_date AS datetime,
                 s.user_agent,
                 ROUND(ST_Length(s.geom::geography)) / 1000 AS length_km,
                 s.computed_h_pixel_density,
                 s.computed_gps_accuracy,
-                t.semantics
+                COALESCE(seq_sem.semantics, '[]'::json) AS semantics
             FROM sequences s
             LEFT JOIN accounts on s.account_id = accounts.id
             LEFT JOIN (
-                    SELECT sequence_id, json_agg(json_strip_nulls(json_build_object(
-                        'key', key,
-                        'value', value
-                    ))) AS semantics
-                    FROM sequences_semantics
-                    GROUP BY sequence_id
-                ) t ON t.sequence_id = s.id
+                SELECT sequence_id, json_agg(json_strip_nulls(json_build_object(
+                    'key', key,
+                    'value', value
+                )) ORDER BY key, value) AS semantics
+                FROM sequences_semantics
+                GROUP BY sequence_id
+            ) seq_sem ON seq_sem.sequence_id = s.id
             WHERE {filter}
             ORDER BY {order1}
             LIMIT {limit}
@@ -184,71 +189,150 @@ def get_collections(request: CollectionsRequest) -> Collections:
 
         # Different request if we want the last n sequences
         #  Useful for paginating from last page to first
-        if request.pagination_filter and (
-            (
-                request.sort_by.fields[0].direction == SQLDirection.ASC
-                and request.pagination_filter.as_string(None).startswith(f"({request.sort_by.fields[0].field.sql_filter.as_string(None)} <")
-            )
-            or (
-                request.sort_by.fields[0].direction == SQLDirection.DESC
-                and request.pagination_filter.as_string(None).startswith(f"({request.sort_by.fields[0].field.sql_filter.as_string(None)} >")
-            )
-        ):
-            base_query = sqlSequencesRaw.format(
-                filter=SQL(" AND ").join(seq_filter),
-                order1=request.sort_by.revert(),
-                limit=request.limit,
-                status=status_field,
-            )
-            sqlSequences = SQL(
+        if request.pagination_filter:
+            # note: we don't want to compare the leading parenthesis
+            pagination = request.pagination_filter.as_string(None).strip("(")
+            first_sort = request.sort_by.fields[0]
+            if (first_sort.direction == SQLDirection.ASC and pagination.startswith(f"{first_sort.field.sql_filter.as_string(None)} <")) or (
+                first_sort.direction == SQLDirection.DESC and pagination.startswith(f"{first_sort.field.sql_filter.as_string(None)} >")
+            ):
+                base_query = sqlSequencesRaw.format(
+                    filter=SQL(" AND ").join(seq_filter),
+                    order1=request.sort_by.revert(),
+                    limit=request.limit,
+                    status=status_field,
+                )
+                sqlSequences = SQL(
+                    """
+                    SELECT *
+                    FROM ({base_query}) s
+                    ORDER BY {order2}
                 """
-                SELECT *
-                FROM ({base_query}) s
-                ORDER BY {order2}
-            """
-            ).format(
-                order2=request.sort_by.as_sql(),
-                base_query=base_query,
-            )
+                ).format(
+                    order2=request.sort_by.as_sql(),
+                    base_query=base_query,
+                )
 
         records = cursor.execute(sqlSequences, seq_params).fetchall()
 
         query_bounds = None
-        for s in records:
-            first_order_val = s.get(request.sort_by.fields[0].field.stac)
-            if first_order_val is None:
-                continue
-            if query_bounds is None:
-                query_bounds = Bounds(first_order_val, first_order_val)
-            else:
-                query_bounds.update(first_order_val)
+        if records:
+            first = [records[0].get(f.field.stac) for f in request.sort_by.fields]
+            last = [records[-1].get(f.field.stac) for f in request.sort_by.fields]
+            query_bounds = Bounds(first, last)
 
         return Collections(
             collections=records,
-            query_first_order_bounds=query_bounds,
+            query_bounds=query_bounds,
         )
 
 
+def get_pagination_stac_filter(sortBy: SortBy, dataBounds: Optional[Bounds[List[Any]]], next: bool) -> str:
+    """Create a pagination API filters, using the sorts and the bounds of the current query"""
+    filters = []
+    bounds = dataBounds.last if next else dataBounds.first
+    for i, f in enumerate(sortBy.fields):
+        direction = f.direction
+        # bounds is a list of values, for all sorty_by fields
+        if (next and direction == SQLDirection.ASC) or (not next and direction == SQLDirection.DESC):
+            cmp = ">"
+        else:
+            cmp = "<"
+        field_pagination = f"{f.field.stac} {cmp} '{bounds[i]}'"
+
+        previous_filters = sortBy.fields[:i]
+        if previous_filters:
+            prev_fields = " AND ".join([f"{f.field.stac} = '{bounds[prev_i]}'" for prev_i, f in enumerate(previous_filters)]) + " AND "
+            filters.append(f"({prev_fields}{field_pagination})")
+        else:
+            filters.append(field_pagination)
+    return " OR ".join(filters)
+
+
+def get_dataset_bounds(
+    conn: psycopg.Connection,
+    sortBy: SortBy,
+    additional_filters: Optional[SQL] = None,
+    additional_filters_params: Optional[Dict[str, Any]] = None,
+) -> Optional[Bounds]:
+    """Computes the dataset bounds from the sortBy field (using lexicographic order)
+
+    if there are several sort-by fields like (inserted_at, updated_at), this will return a bound with minimum (resp maximum)
+    inserted_at value, and for this value, the minimum (resp maximum) updated_at value.
+    """
+    with conn.cursor() as cursor:
+
+        sql_bounds = cursor.execute(
+            SQL(
+                """WITH min_bounds AS (
+    SELECT {fields} from sequences s WHERE {filters} ORDER BY {ordered_fields} LIMIT 1
+),
+max_bounds AS (
+    SELECT {fields} from sequences s WHERE {filters} ORDER BY {reverse_fields} LIMIT 1
+)
+SELECT * FROM min_bounds, max_bounds;
+            """
+            ).format(
+                fields=SQL(", ").join([f.field.sql_column for f in sortBy.fields]),
+                ordered_fields=sortBy.as_non_aliased_sql(),
+                reverse_fields=sortBy.revert_non_aliased_sql(),
+                filters=additional_filters or SQL("TRUE"),
+            ),
+            params=additional_filters_params or {},
+        ).fetchone()
+        if not sql_bounds:
+            return None
+        min = [sql_bounds[i] for i, f in enumerate(sortBy.fields)]
+        max = [sql_bounds[i + len(sortBy.fields)] for i, f in enumerate(sortBy.fields)]
+        return Bounds(first=min, last=max)
+
+
+def has_previous_results(sortBy: SortBy, datasetBounds: Bounds, dataBounds: Bounds) -> bool:
+    """Check if there are results in the database before the one returned by the queries
+    To do this, we do a lexicographic comparison of the bounds, using the fields direction
+
+    Note: the bounds are reversed for the DESC direction, so the bounds.min >= bounds.max for the DESC direction
+    """
+    for i, f in enumerate(sortBy.fields):
+        if dataBounds.first[i] is None or datasetBounds.first[i] is None:
+            continue
+        if (f.direction == SQLDirection.ASC and dataBounds.first[i] > datasetBounds.first[i]) or (
+            f.direction == SQLDirection.DESC and datasetBounds.first[i] > dataBounds.first[i]
+        ):
+            return True
+    return False
+
+
+def has_next_results(sortBy: SortBy, datasetBounds: Bounds, dataBounds: Bounds) -> bool:
+    """Check if there are results in the database after the one returned by the queries
+    To do this, we do a lexicographic comparison of the bounds, using the fields direction"""
+    for i, f in enumerate(sortBy.fields):
+        if dataBounds.last[i] is None or datasetBounds.last[i] is None:
+            continue
+        if (f.direction == SQLDirection.ASC and dataBounds.last[i] < datasetBounds.last[i]) or (
+            f.direction == SQLDirection.DESC and datasetBounds.last[i] < dataBounds.last[i]
+        ):
+            return True
+    return False
+
+
 def get_pagination_links(
     route: str,
     routeArgs: dict,
-    field: str,
-    direction: SQLDirection,
+    sortBy: SortBy,
     datasetBounds: Bounds,
     dataBounds: Optional[Bounds],
     additional_filters: Optional[str],
 ) -> List:
     """Computes STAC links to handle pagination"""
 
-    sortby = f"{'+' if direction == SQLDirection.ASC else '-'}{field}"
+    sortby = sortBy.as_stac()
     links = []
-    if dataBounds is None:
+    if dataBounds is None or datasetBounds is None:
         return links
 
     # Check if first/prev links are necessary
-    if (direction == SQLDirection.ASC and datasetBounds.min < dataBounds.min) or (
-        direction == SQLDirection.DESC and dataBounds.max < datasetBounds.max
-    ):
+    if has_previous_results(sortBy, datasetBounds=datasetBounds, dataBounds=dataBounds):
         links.append(
             {
                 "rel": "first",
@@ -256,7 +340,9 @@ def get_pagination_links(
                 "href": url_for(route, _external=True, **routeArgs, filter=additional_filters, sortby=sortby),
             }
         )
-        page_filter = f"{field} {'<' if direction == SQLDirection.ASC else '>'} '{dataBounds.min if direction == SQLDirection.ASC else dataBounds.max}'"
+
+        page_filter = get_pagination_stac_filter(sortBy, dataBounds, next=False)
+
         links.append(
             {
                 "rel": "prev",
@@ -273,11 +359,8 @@ def get_pagination_links(
         )
 
     # Check if next/last links are required
-    if (direction == SQLDirection.ASC and dataBounds.max < datasetBounds.max) or (
-        direction == SQLDirection.DESC and datasetBounds.min < dataBounds.min
-    ):
-        next_filter = f"{field} {'>' if direction == SQLDirection.ASC else '<'} '{dataBounds.max if direction == SQLDirection.ASC else dataBounds.min}'"
-        last_filter = f"{field} {'<=' if direction == SQLDirection.ASC else '>='} '{datasetBounds.max if direction == SQLDirection.ASC else datasetBounds.min}'"
+    if has_next_results(sortBy, datasetBounds=datasetBounds, dataBounds=dataBounds):
+        next_filter = get_pagination_stac_filter(sortBy, dataBounds, next=True)
         links.append(
             {
                 "rel": "next",
@@ -292,6 +375,10 @@ def get_pagination_links(
                 ),
             }
         )
+        # for last, we only consider the first field used for sorting, the rest are useless
+        # Note: we compare to the datasetBounds last since it depends on the sort direction (so, for DESC, it last<first)
+        f = sortBy.fields[0]
+        last_filter = f"{f.field.stac} {'<=' if f.direction == SQLDirection.ASC else '>='} '{datasetBounds.last[0]}'"
         links.append(
             {
                 "rel": "last",
@@ -341,7 +428,7 @@ def sort_collection(db, collectionId: UUID, sortby: CollectionSort):
     """
     Sort a collection by a given parameter
 
-    Note: the transaction is not commited at the end, you need to commit it or use an autocommit connection
+    Note: the transaction is not committed at the end, you need to commit it or use an autocommit connection
     """
 
     # Remove existing order, and keep list of pictures IDs
@@ -423,23 +510,18 @@ def sort_collection(db, collectionId: UUID, sortby: CollectionSort):
     picForDb = [(collectionId, i + 1, p["id"]) for i, p in enumerate(picMetas)]
 
     # Inject back pictures in sequence
-    db.executemany(
-        SQL(
-            """
-            INSERT INTO sequences_pictures(seq_id, rank, pic_id)
-            VALUES (%s, %s, %s)
-            """
-        ),
-        picForDb,
-    )
+    db.executemany(SQL("INSERT INTO sequences_pictures(seq_id, rank, pic_id) VALUES (%s, %s, %s)"), picForDb)
+
+    # we update the geometry of the sequence after this (the other computed fields have no need for an update)
+    db.execute(SQL("UPDATE sequences SET geom = compute_sequence_geom(id) WHERE id = %s"), [collectionId])
 
 
 def update_headings(
     db,
     sequenceId: UUID,
     editingAccount: Optional[UUID] = None,
-    relativeHeading: int = 0,
-    updateOnlyMissing: bool = True,
+    relativeHeading: Optional[int] = None,
+    updateOnlyMissing: Optional[bool] = None,
 ):
     """Defines pictures heading according to sequence path.
     Database is not committed in this function, to make entry definitively stored
@@ -451,27 +533,42 @@ def update_headings(
             Database connection
     sequenceId : uuid
             The sequence's uuid, as stored in the database
-    relativeHeading : int
+    relativeHeading : Optional[int]
             Camera relative orientation compared to path, in degrees clockwise.
             Example: 0° = looking forward, 90° = looking to right, 180° = looking backward, -90° = looking left.
-    updateOnlyMissing : bool
+            If not provided, will first use the relative_heading stored in the sequence's metadata, then the relative_heading of its upload_set (if if none is set, default to 0).
+    updateOnlyMissing : Optional[bool]
             If true, doesn't change existing heading values in database
+            if not provided, we check if some relative heading has been set (either in the sequence or in its upload_set), and if so, we recompute all
     """
-
     db.execute(
         SQL(
-            """
-		WITH h AS (
+            """WITH
+        relative_heading AS (
+            SELECT COALESCE(
+                    %(relativeHeading)s, 
+                    (SELECT (metadata->>'relative_heading')::int FROM sequences WHERE id = %(seq)s),
+                    (SELECT upload_sets.relative_heading FROM sequences JOIN upload_sets ON sequences.upload_set_id = upload_sets.id WHERE sequences.id = %(seq)s),
+                    0
+            ) AS heading,
+            COALESCE(
+                %(update_only_missing)s,
+                (SELECT metadata->'relative_heading' IS NULL FROM sequences WHERE id = %(seq)s and metadata ? 'relative_heading'),
+                (SELECT upload_sets.relative_heading IS NULL FROM sequences JOIN upload_sets ON sequences.upload_set_id = upload_sets.id WHERE sequences.id = %(seq)s)
+            ) AS update_only_missing
+        )
+        , h AS (
 			SELECT
 				p.id,
                 p.heading AS old_heading,
 				CASE
 					WHEN LEAD(sp.rank) OVER othpics IS NULL AND LAG(sp.rank) OVER othpics IS NULL
-						THEN NULL
+                        -- if there is a single picture, we take the relative heading directly
+						THEN (SELECT heading FROM relative_heading)
 					WHEN LEAD(sp.rank) OVER othpics IS NULL
-						THEN (360 + FLOOR(DEGREES(ST_Azimuth(LAG(p.geom) OVER othpics, p.geom)))::int + (%(diff)s %% 360)) %% 360
+						THEN (360 + FLOOR(DEGREES(ST_Azimuth(LAG(p.geom) OVER othpics, p.geom)))::int + ((SELECT heading FROM relative_heading) %% 360)) %% 360
 					ELSE
-						(360 + FLOOR(DEGREES(ST_Azimuth(p.geom, LEAD(p.geom) OVER othpics)))::int + (%(diff)s %% 360)) %% 360
+						(360 + FLOOR(DEGREES(ST_Azimuth(p.geom, LEAD(p.geom) OVER othpics)))::int + ((SELECT heading FROM relative_heading) %% 360)) %% 360
 				END AS heading
 			FROM pictures p
 			JOIN sequences_pictures sp ON sp.pic_id = p.id AND sp.seq_id = %(seq)s
@@ -480,13 +577,15 @@ def update_headings(
 		UPDATE pictures p
 		SET heading = h.heading, heading_computed = true {editing_account}
 		FROM h
-		WHERE h.id = p.id {update_missing}
+		WHERE h.id = p.id AND (
+            (SELECT NOT update_only_missing FROM relative_heading) 
+            OR (p.heading IS NULL OR p.heading = 0 OR p.heading_computed) -- # lots of camera have heading set to 0 for unset heading, so we recompute the heading when it's 0 too, even if this could be a valid value
+        )
 		"""
         ).format(
-            update_missing=SQL(" AND (p.heading IS NULL OR p.heading = 0 OR p.heading_computed)") if updateOnlyMissing else SQL(""),
             editing_account=SQL(", last_account_to_edit = %(account)s") if editingAccount is not None else SQL(""),
-        ),  # lots of camera have heading set to 0 for unset heading, so we recompute the heading when it's 0 too, even if this could be a valid value
-        {"seq": sequenceId, "diff": relativeHeading, "account": editingAccount},
+        ),
+        {"seq": sequenceId, "relativeHeading": relativeHeading, "account": editingAccount, "update_only_missing": updateOnlyMissing},
     )
 
 
@@ -512,14 +611,13 @@ def finalize(cursor, seqId: UUID, logger: logging.Logger = logging.getLogger()):
         span.set_data("sequence_id", seqId)
         logger.debug(f"Finalizing sequence {seqId}")
 
-        with utils.time.log_elapsed(f"Finalizing sequence {seqId}"):
-            # Complete missing headings in pictures
-            update_headings(cursor, seqId)
+        # Complete missing headings in pictures
+        update_headings(cursor, seqId)
 
-            # Change sequence database status in DB
-            # Also generates data in computed columns
-            cursor.execute(
-                """WITH
+        # Change sequence database status in DB
+        # Also generates data in computed columns
+        cursor.execute(
+            """WITH
 aggregated_pictures AS (
 SELECT
     sp.seq_id, 
@@ -548,10 +646,10 @@ computed_gps_accuracy = gpsacc
 FROM aggregated_pictures
 WHERE id = %(seq)s
             """,
-                {"seq": seqId},
-            )
+            {"seq": seqId},
+        )
 
-            logger.info(f"Sequence {seqId} is ready")
+        logger.info(f"Sequence {seqId} is ready")
 
 
 def update_pictures_grid() -> Optional[datetime.datetime]:

geovisio/utils/tags.py

@@ -0,0 +1,31 @@
+from enum import Enum
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class TagAction(str, Enum):
+    """Actions to perform on a tag list"""
+
+    add = "add"
+    delete = "delete"
+
+
+class SemanticTagUpdate(BaseModel):
+    """Parameters used to update a tag list"""
+
+    action: TagAction = Field(default=TagAction.add)
+    """Action to perform on the tag list. The default action is `add` which will add the given tag to the list.
+    The action can also be to `delete` the key/value"""
+    key: str = Field(max_length=256)
+    """Key of the tag to update limited to 256 characters"""
+    value: str = Field(max_length=2048)
+    """Value of the tag to update limited ot 2048 characters"""
+
+    model_config = ConfigDict(use_attribute_docstrings=True)
+
+
+class SemanticTag(BaseModel):
+    key: str
+    """Key of the tag"""
+    value: str
+    """Value of the tag"""