geopic-tag-reader 1.7.0__tar.gz → 1.8.0__tar.gz

{geopic_tag_reader-1.7.0 → geopic_tag_reader-1.8.0}/CHANGELOG.md

@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.8.0] - 2025-05-25
+
+### Changed
+
+- Improved duplicates detection, now the whole sequence is deduplicated, not only the consecutive pictures
+- **Breaking Change** Now the `duplicate_pictures` in the report is a list of `Duplicate` (instead of `Picture`), so we can get more information about the duplicate.
+
 ## [1.7.0] - 2025-05-25
 
 ### Changed
@@ -291,7 +298,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - EXIF tag reading methods extracted from [Panoramax/GeoVisio API](https://gitlab.com/panoramax/server/api)
 
-[Unreleased]: https://gitlab.com/panoramax/server/geo-picture-tag-reader/-/compare/1.7.0...main
+[Unreleased]: https://gitlab.com/panoramax/server/geo-picture-tag-reader/-/compare/1.8.0...main
+[1.8.0]: https://gitlab.com/panoramax/server/geo-picture-tag-reader/-/compare/1.7.0...1.8.0
 [1.7.0]: https://gitlab.com/panoramax/server/geo-picture-tag-reader/-/compare/1.6.0...1.7.0
 [1.6.0]: https://gitlab.com/panoramax/server/geo-picture-tag-reader/-/compare/1.5.0...1.6.0
 [1.5.0]: https://gitlab.com/panoramax/server/geo-picture-tag-reader/-/compare/1.4.2...1.5.0

{geopic_tag_reader-1.7.0 → geopic_tag_reader-1.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: geopic-tag-reader
-Version: 1.7.0
+Version: 1.8.0
 Summary: GeoPicTagReader
 Author-email: Adrien PAVIE <panieravide@riseup.net>
 Requires-Python: >=3.9
@@ -14,6 +14,7 @@ Requires-Dist: timezonefinder == 6.5.9
 Requires-Dist: pytz ~= 2025.2
 Requires-Dist: types-pytz ~= 2025.2.0
 Requires-Dist: types-python-dateutil ~= 2.9.0
+Requires-Dist: rtree ~= 1.4.0
 Requires-Dist: flit ~= 3.8.0 ; extra == "build"
 Requires-Dist: black ~= 25.1.0 ; extra == "dev"
 Requires-Dist: mypy ~= 1.15.0 ; extra == "dev"

{geopic_tag_reader-1.7.0 → geopic_tag_reader-1.8.0}/geopic_tag_reader/__init__.py

@@ -2,4 +2,4 @@
 GeoPicTagReader
 """
 
-__version__ = "1.7.0"
+__version__ = "1.8.0"

{geopic_tag_reader-1.7.0 → geopic_tag_reader-1.8.0}/geopic_tag_reader/sequence.py

@@ -1,9 +1,10 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from enum import Enum
 from typing import Optional, List, Tuple
 from pathlib import PurePath
 from geopic_tag_reader.reader import GeoPicTags
 import datetime
+from rtree import index
 import math
 
 
@@ -37,6 +38,7 @@ class SplitParams:
 class Picture:
     filename: str
     metadata: GeoPicTags
+    heading_computed: bool = False
 
     def distance_to(self, other) -> float:
         """Computes distance in meters based on Haversine formula"""
@@ -50,7 +52,7 @@ class Picture:
         distance = R * c
         return distance
 
-    def rotation_angle(self, other) -> Optional[float]:
+    def rotation_angle(self, other) -> Optional[int]:
         return rotation_angle(self.metadata.heading, other.metadata.heading)
 
 
@@ -116,11 +118,19 @@ class Sequence:
         return (otherSeq.from_ts() - self.to_ts(), otherSeq.pictures[0].distance_to(self.pictures[-1]))  # type: ignore
 
 
+@dataclass
+class Duplicate:
+    picture: Picture
+    duplicate_of: Picture
+    distance: float
+    angle: Optional[int]
+
+
 @dataclass
 class DispatchReport:
     sequences: List[Sequence]
-    duplicate_pictures: Optional[List[Picture]] = None
-    sequences_splits: Optional[List[Split]] = None
+    duplicate_pictures: List[Duplicate] = field(default_factory=list)
+    sequences_splits: List[Split] = field(default_factory=list)
 
 
 def sort_pictures(pictures: List[Picture], method: Optional[SortMethod] = SortMethod.time_asc) -> List[Picture]:
@@ -202,7 +212,38 @@ def sort_pictures(pictures: List[Picture], method: Optional[SortMethod] = SortMe
     return pictures
 
 
-def find_duplicates(pictures: List[Picture], params: Optional[MergeParams] = None) -> Tuple[List[Picture], List[Picture]]:
+def are_duplicates(a: Picture, b: Picture, params: MergeParams) -> Optional[Tuple[float, Optional[int]]]:
+    """
+    Check if 2 pictures are too similar and should be considered duplicates
+
+    They are duplicates if they are close to each other, and for non 360 pictures, if they are roughly in the same direction.
+
+    Note that we only consider the direction (also called heading) if it is provided by the camera (and not computed with the sequences geometries)
+    since GPS can drift a bit resulting in erratic direction when waiting at a traffic light cf https://gitlab.com/panoramax/server/api/-/issues/231#note_2329723526
+
+    Return None if not duplicates, or the distance/angle if they are
+    """
+    dist = a.distance_to(b)
+
+    if params.maxDistance is None or dist > params.maxDistance:
+        return None
+
+    # Compare angle (if available on both images)
+    angle = a.rotation_angle(b)
+    # if one of the heading has been computed, we cannot rely on this angle being correct, so we don't consider it for the deduplication
+    # it's especially important when stopped and the GPS drift a bit, cf https://gitlab.com/panoramax/server/api/-/issues/231#note_2329723526
+    angle_computed = b.heading_computed or a.heading_computed
+    if angle is None or angle_computed or params.maxRotationAngle is None:
+        return (dist, None)
+    if angle <= params.maxRotationAngle:
+        return (dist, angle)
+    return None
+
+
+APPROX_DEGREE_TO_METER = 0.00001  # this is roughly 1m
+
+
+def find_duplicates(pictures: List[Picture], params: Optional[MergeParams] = None) -> Tuple[List[Picture], List[Duplicate]]:
     """
     Finds too similar pictures.
     Note that input list should be properly sorted.
@@ -217,42 +258,51 @@ def find_duplicates(pictures: List[Picture], params: Optional[MergeParams] = Non
     (Non-duplicates pictures, Duplicates pictures)
     """
 
-    if params is None or not params.is_merge_needed():
+    if params is None or not params.is_merge_needed() or not pictures:
         return (pictures, [])
+    assert params.maxDistance
 
     nonDups: List[Picture] = []
-    dups: List[Picture] = []
-    lastNonDuplicatedPicId = 0
+    duplicates = []
+    duplicates_idx = set()
 
-    for i, currentPic in enumerate(pictures):
-        if i == 0:
-            nonDups.append(currentPic)
-            continue
+    rtree_index = index.Index(((i, (p.metadata.lon, p.metadata.lat, p.metadata.lon, p.metadata.lat), None) for i, p in enumerate(pictures)))
 
-        prevPic = pictures[lastNonDuplicatedPicId]
+    # the rtree will give us all the neighbors in an approximated bounding box,
+    # and will check, for all those pictures if some pictures are really closed, using a real haversine distance
+    # we do a rough conversion between the maxDistance (in m) to degree, since it's only for the initial bounding box
+    # and we use a bbox bigger than necessary (could be half by direction) to not miss duplicates due to the degree to meter approximation
 
-        if prevPic.metadata is None or currentPic.metadata is None:
-            nonDups.append(currentPic)
+    bounding_box_tolerance_approx = params.maxDistance * APPROX_DEGREE_TO_METER
+    for i, currentPic in enumerate(pictures):
+        if i in duplicates_idx:
+            # the picture has already been flagged as duplicate by one of its neighbor, we can skip it
             continue
 
-        is_duplicate = False
+        bounding_box = (
+            currentPic.metadata.lon - bounding_box_tolerance_approx,
+            currentPic.metadata.lat - bounding_box_tolerance_approx,
+            currentPic.metadata.lon + bounding_box_tolerance_approx,
+            currentPic.metadata.lat + bounding_box_tolerance_approx,
+        )
 
-        # Compare distance
-        dist = prevPic.distance_to(currentPic)
+        near_pics_idx = rtree_index.nearest(bounding_box, num_results=100, objects=False)
 
-        if params.maxDistance is not None and dist <= params.maxDistance:
-            # Compare angle (if available on both images)
-            angle = prevPic.rotation_angle(currentPic)
-            if angle is None or params.maxRotationAngle is None or angle <= params.maxRotationAngle:
-                is_duplicate = True
+        for neighbor_idx in near_pics_idx:
+            if neighbor_idx == i:
+                continue
+            if neighbor_idx in duplicates_idx:
+                continue
+            neighbor = pictures[neighbor_idx]
+            duplicate_details = are_duplicates(currentPic, neighbor, params)
+            if duplicate_details:
+                distance, angle = duplicate_details
+                duplicates_idx.add(neighbor_idx)
+                duplicates.append(Duplicate(picture=neighbor, duplicate_of=currentPic, distance=round(distance, 2), angle=angle))
 
-        if is_duplicate:
-            dups.append(currentPic)
-        else:
-            lastNonDuplicatedPicId = i
-            nonDups.append(currentPic)
+        nonDups.append(currentPic)
 
-    return (nonDups, dups)
+    return (nonDups, duplicates)
 
 
 def split_in_sequences(pictures: List[Picture], splitParams: Optional[SplitParams] = SplitParams()) -> Tuple[List[Sequence], List[Split]]:
@@ -354,10 +404,14 @@ def dispatch_pictures(
     # Sort
     myPics = sort_pictures(pictures, sortMethod)
 
-    # De-duplicate
-    (myPics, dupsPics) = find_duplicates(myPics, mergeParams)
-
     # Split in sequences
     (mySeqs, splits) = split_in_sequences(myPics, splitParams)
 
-    return DispatchReport(mySeqs, dupsPics if len(dupsPics) > 0 else None, splits if len(splits) > 0 else None)
+    # De-duplicate inside each sequences
+    dups_pics = []
+    for s in mySeqs:
+        (myPics, dups) = find_duplicates(s.pictures, mergeParams)
+        s.pictures = myPics
+        dups_pics.extend(dups)
+
+    return DispatchReport(sequences=mySeqs, duplicate_pictures=dups_pics, sequences_splits=splits)

{geopic_tag_reader-1.7.0 → geopic_tag_reader-1.8.0}/geopic_tag_reader/translations/en/LC_MESSAGES/geopic_tag_reader.po

@@ -7,8 +7,8 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2025-08-25 15:13+0200\n"
-"PO-Revision-Date: 2025-08-25 15:13+0200\n"
+"POT-Creation-Date: 2025-08-25 15:35+0200\n"
+"PO-Revision-Date: 2025-08-25 15:35+0200\n"
 "Last-Translator: Automatically generated\n"
 "Language-Team: none\n"
 "Language: en\n"

{geopic_tag_reader-1.7.0 → geopic_tag_reader-1.8.0}/geopic_tag_reader/translations/geopic_tag_reader.pot

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2025-08-25 15:13+0200\n"
+"POT-Creation-Date: 2025-08-25 15:35+0200\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

{geopic_tag_reader-1.7.0 → geopic_tag_reader-1.8.0}/pyproject.toml

@@ -15,6 +15,7 @@ dependencies = [
     "pytz ~= 2025.2",
     "types-pytz ~= 2025.2.0",
     "types-python-dateutil ~= 2.9.0",
+    "rtree ~= 1.4.0",
 ]
 requires-python = ">=3.9"