umap-project 2.7.3__py3-none-any.whl → 2.8.0a0__py3-none-any.whl

umap/storage.py

@@ -1,9 +1,17 @@
+import operator
+import os
+import shutil
+import time
+from gzip import GzipFile
 from pathlib import Path
 
+from botocore.exceptions import ClientError
 from django.conf import settings
 from django.contrib.staticfiles.storage import ManifestStaticFilesStorage
+from django.core.files.storage import FileSystemStorage
 from rcssmin import cssmin
 from rjsmin import jsmin
+from storages.backends.s3 import S3Storage
 
 
 class UmapManifestStaticFilesStorage(ManifestStaticFilesStorage):
@@ -62,3 +70,147 @@ class UmapManifestStaticFilesStorage(ManifestStaticFilesStorage):
                     minified = cssmin(initial)
                     path.write_text(minified)
             yield original_path, processed_path, True
+
+
+class UmapS3(S3Storage):
+    gzip = True
+
+    def get_reference_version(self, instance):
+        metadata = self.connection.meta.client.head_object(
+            Bucket=self.bucket_name, Key=instance.geojson.name
+        )
+        # Do not fail if bucket does not handle versioning
+        return metadata.get("VersionId", metadata["ETag"])
+
+    def make_filename(self, instance):
+        return f"{str(instance.pk)}.geojson"
+
+    def list_versions(self, instance):
+        response = self.connection.meta.client.list_object_versions(
+            Bucket=self.bucket_name, Prefix=instance.geojson.name
+        )
+        return [
+            {
+                "ref": version["VersionId"],
+                "at": version["LastModified"].timestamp() * 1000,
+                "size": version["Size"],
+            }
+            for version in response["Versions"]
+        ]
+
+    def get_version(self, ref, instance):
+        try:
+            data = self.connection.meta.client.get_object(
+                Bucket=self.bucket_name,
+                Key=instance.geojson.name,
+                VersionId=ref,
+            )
+        except ClientError:
+            raise ValueError(f"Invalid version reference: {ref}")
+        return GzipFile(mode="r", fileobj=data["Body"]).read()
+
+    def get_version_path(self, ref, instance):
+        return self.url(instance.geojson.name, parameters={"VersionId": ref})
+
+    def onDatalayerSave(self, instance):
+        pass
+
+    def onDatalayerDelete(self, instance):
+        return self.connection.meta.client.delete_object(
+            Bucket=self.bucket_name,
+            Key=instance.geojson.name,
+        )
+
+
+class UmapFileSystem(FileSystemStorage):
+    def get_reference_version(self, instance):
+        return self._extract_version_ref(instance.geojson.name)
+
+    def make_filename(self, instance):
+        root = self._base_path(instance)
+        name = "%s_%s.geojson" % (instance.pk, int(time.time() * 1000))
+        return root / name
+
+    def list_versions(self, instance):
+        root = self._base_path(instance)
+        names = self.listdir(root)[1]
+        names = [name for name in names if self._is_valid_version(name, instance)]
+        versions = [self._version_metadata(name, instance) for name in names]
+        versions.sort(reverse=True, key=operator.itemgetter("at"))
+        return versions
+
+    def get_version(self, ref, instance):
+        with self.open(self.get_version_path(ref, instance), "r") as f:
+            return f.read()
+
+    def get_version_path(self, ref, instance):
+        base_path = Path(settings.MEDIA_ROOT) / self._base_path(instance)
+        fullpath = base_path / f"{instance.pk}_{ref}.geojson"
+        if instance.old_id and not fullpath.exists():
+            fullpath = base_path / f"{instance.old_id}_{ref}.geojson"
+        if not fullpath.exists():
+            raise ValueError(f"Invalid version reference: {ref}")
+        return fullpath
+
+    def onDatalayerSave(self, instance):
+        self._purge_gzip(instance)
+        self._purge_old_versions(instance, keep=settings.UMAP_KEEP_VERSIONS)
+
+    def onDatalayerDelete(self, instance):
+        self._purge_gzip(instance)
+        self._purge_old_versions(instance, keep=None)
+
+    def _extract_version_ref(self, path):
+        version = path.split(".")[0]
+        if "_" in version:
+            return version.split("_")[-1]
+        return version
+
+    def _base_path(self, instance):
+        path = ["datalayer", str(instance.map.pk)[-1]]
+        if len(str(instance.map.pk)) > 1:
+            path.append(str(instance.map.pk)[-2])
+        path.append(str(instance.map.pk))
+        return Path(os.path.join(*path))
+
+    def _is_valid_version(self, name, instance):
+        valid_prefixes = [name.startswith("%s_" % instance.pk)]
+        if instance.old_id:
+            valid_prefixes.append(name.startswith("%s_" % instance.old_id))
+        return any(valid_prefixes) and name.endswith(".geojson")
+
+    def _version_metadata(self, name, instance):
+        ref = self._extract_version_ref(name)
+        return {
+            "name": name,
+            "ref": ref,
+            "at": ref,
+            "size": self.size(self._base_path(instance) / name),
+        }
+
+    def _purge_old_versions(self, instance, keep=None):
+        root = self._base_path(instance)
+        versions = self.list_versions(instance)
+        if keep is not None:
+            versions = versions[keep:]
+        for version in versions:
+            name = version["name"]
+            # Should not be in the list, but ensure to not delete the file
+            # currently used in database
+            if keep is not None and instance.geojson.name.endswith(name):
+                continue
+            try:
+                self.delete(root / name)
+            except FileNotFoundError:
+                pass
+
+    def _purge_gzip(self, instance):
+        root = self._base_path(instance)
+        names = self.listdir(root)[1]
+        prefixes = [f"{instance.pk}_"]
+        if instance.old_id:
+            prefixes.append(f"{instance.old_id}_")
+        prefixes = tuple(prefixes)
+        for name in names:
+            if name.startswith(prefixes) and name.endswith(".gz"):
+                self.delete(root / name)

umap/templates/registration/login.html

@@ -19,10 +19,11 @@
     <header class="umap-nav">
       {% include "umap/branding.html" %}
     </header>
+    <h2>{% trans "To save and easily find your maps, identify yourself." %}</h2>
     {% if ENABLE_ACCOUNT_LOGIN %}
-      <h2>
-        {% trans "Please log in with your account" %}
-      </h2>
+      <h3>
+        {% trans "Please log in with your account:" %}
+      </h3>
       <div>
         {% if form.non_field_errors %}
           <ul class="form-errors">
@@ -47,9 +48,9 @@
       </div>
     {% endif %}
     {% if backends.backends|length %}
-      <h2>
-        {% trans "Please choose a provider" %}
-      </h2>
+      <h3>
+        {% trans "Please choose a provider:" %}
+      </h3>
       <div>
         <ul class="login-grid block-grid">
           {% for name in backends.backends %}

umap/templates/umap/css.html

@@ -24,6 +24,8 @@
 <link rel="stylesheet" href="{% static 'umap/font.css' %}" />
 <link rel="stylesheet" href="{% static 'umap/css/icon.css' %}" />
 <link rel="stylesheet" href="{% static 'umap/base.css' %}" />
+<link rel="stylesheet" href="{% static 'umap/css/popup.css' %}" />
+<link rel="stylesheet" href="{% static 'umap/css/form.css' %}" />
 <link rel="stylesheet" href="{% static 'umap/content.css' %}" />
 <link rel="stylesheet" href="{% static 'umap/nav.css' %}" />
 <link rel="stylesheet" href="{% static 'umap/map.css' %}" />
@@ -35,4 +37,5 @@
 <link rel="stylesheet" href="{% static 'umap/css/dialog.css' %}" />
 <link rel="stylesheet" href="{% static 'umap/css/importers.css' %}" />
 <link rel="stylesheet" href="{% static 'umap/css/tableeditor.css' %}" />
+<link rel="stylesheet" href="{% static 'umap/css/bar.css' %}" />
 <link rel="stylesheet" href="{% static 'umap/theme.css' %}" />

umap/templates/umap/js.html

@@ -41,5 +41,4 @@
 <script src="{% static 'umap/js/umap.core.js' %}" defer></script>
 <script src="{% static 'umap/js/umap.forms.js' %}" defer></script>
 <script src="{% static 'umap/js/umap.controls.js' %}" defer></script>
-<script src="{% static 'umap/js/umap.js' %}" defer></script>
-<script src="{% static 'umap/js/components/fragment.js' %}" defer></script>
+<script type="module" src="{% static 'umap/js/components/fragment.js' %}" defer></script>

umap/templates/umap/map_init.html

@@ -1,12 +1,11 @@
-{% load umap_tags %}
+{% load umap_tags static %}
 
 {% include "umap/messages.html" %}
 <div id="map">
 </div>
 <!-- djlint:off -->
-<script defer type="text/javascript">
-  window.addEventListener('DOMContentLoaded', (event) => {
-    U.MAP = new U.Map("map", {{ map_settings|notag|safe }})
-  })
+<script defer type="module">
+    import Umap from '{% static "umap/js/modules/umap.js" %}'
+    U.MAP = new Umap("map", {{ map_settings|notag|safe }})
 </script>
 <!-- djlint:on -->

umap/templates/umap/user_dashboard.html

@@ -1,6 +1,6 @@
 {% extends "umap/content.html" %}
 
-{% load i18n %}
+{% load i18n static %}
 
 {% block head_title %}
   {{ SITE_NAME }} - {% trans "My Dashboard" %}
@@ -45,23 +45,22 @@
 {% endblock maincontent %}
 {% block bottom_js %}
   {{ block.super }}
-  <script type="text/javascript">
-    !(function () {
-      const CACHE = {}
-      for (const mapOpener of document.querySelectorAll("button.map-opener")) {
-        mapOpener.addEventListener('click', (event) => {
-          const button = event.target.closest('button')
-          button.nextElementSibling.showModal()
-          const mapId = button.dataset.mapId
-          if (!document.querySelector(`#${mapId}_target`).children.length) {
-            const previewSettings = JSON.parse(document.getElementById(mapId).textContent)
-            const map = new U.Map(`${mapId}_target`, previewSettings)
-            CACHE[mapId] = map
-          } else {
-            CACHE[mapId].invalidateSize()
-          }
-        })
-      }
-    })()
+  <script type="module">
+    import Umap from '{% static "umap/js/modules/umap.js" %}'
+    const CACHE = {}
+    for (const mapOpener of document.querySelectorAll("button.map-opener")) {
+      mapOpener.addEventListener('click', (event) => {
+        const button = event.target.closest('button')
+        button.nextElementSibling.showModal()
+        const mapId = button.dataset.mapId
+        if (!document.querySelector(`#${mapId}_target`).children.length) {
+          const previewSettings = JSON.parse(document.getElementById(mapId).textContent)
+          const map = new Umap(`${mapId}_target`, previewSettings)
+          CACHE[mapId] = map
+        } else {
+          CACHE[mapId].invalidateSize()
+        }
+      })
+    }
   </script>
 {% endblock bottom_js %}

umap/tests/base.py

@@ -1,5 +1,6 @@
 import copy
 import json
+import uuid
 from pathlib import Path
 
 import factory
@@ -41,7 +42,7 @@ class LicenceFactory(factory.django.DjangoModelFactory):
 
 class TileLayerFactory(factory.django.DjangoModelFactory):
     name = "Test zoom layer"
-    url_template = "https://{s}.test.org/osmfr/{z}/{x}/{y}.png"
+    url_template = "https://tile.openstreetmap.org/{z}/{x}/{y}.png"
     attribution = "Test layer attribution"
 
     class Meta:
@@ -101,6 +102,7 @@ class MapFactory(factory.django.DjangoModelFactory):
 
     licence = factory.SubFactory(LicenceFactory)
     owner = factory.SubFactory(UserFactory)
+    share_status = Map.PUBLIC
 
     @classmethod
     def _adjust_kwargs(cls, **kwargs):
@@ -114,6 +116,7 @@ class MapFactory(factory.django.DjangoModelFactory):
 
 
 class DataLayerFactory(factory.django.DjangoModelFactory):
+    uuid = factory.LazyFunction(lambda: uuid.uuid4())
     map = factory.SubFactory(MapFactory)
     name = "test datalayer"
     description = "test description"
@@ -154,5 +157,6 @@ def login_required(response):
 
 
 def mock_tiles(route):
+    print("Intercepted route", route.request.url)
     path = Path(__file__).parent / "fixtures/empty_tile.png"
     route.fulfill(path=path)

umap/tests/integration/conftest.py

@@ -1,4 +1,5 @@
 import os
+import re
 import subprocess
 import time
 from pathlib import Path
@@ -25,7 +26,7 @@ def set_timeout(context):
 @pytest.fixture(autouse=True)
 def mock_osm_tiles(page):
     if not bool(os.environ.get("PWDEBUG", False)):
-        page.route("*/**/osmfr/**", mock_tiles)
+        page.route(re.compile(r".*tile\..*"), mock_tiles)
 
 
 @pytest.fixture

umap/tests/integration/test_anonymous_owned_map.py

@@ -76,8 +76,6 @@ def test_owner_permissions_form(map, datalayer, live_server, owner_session):
     edit_permissions = owner_session.get_by_title("Update permissions and editors")
     expect(edit_permissions).to_be_visible()
     edit_permissions.click()
-    select = owner_session.locator(".umap-field-share_status select")
-    expect(select).to_be_hidden()
     owner_field = owner_session.locator(".umap-field-owner")
     expect(owner_field).to_be_hidden()
     editors_field = owner_session.locator(".umap-field-editors input")
@@ -92,8 +90,15 @@ def test_owner_permissions_form(map, datalayer, live_server, owner_session):
         ".datalayer-permissions select[name='edit_status'] option:checked"
     )
     expect(option).to_have_text("Inherit")
-    # Those fields should not be present in anonymous maps
-    expect(owner_session.locator(".umap-field-share_status select")).to_be_hidden()
+    expect(owner_session.locator(".umap-field-share_status select")).to_be_visible()
+    options = [
+        int(option.get_attribute("value"))
+        for option in owner_session.locator(
+            ".umap-field-share_status select option"
+        ).all()
+    ]
+    assert options == [Map.DRAFT, Map.PUBLIC]
+    # This field should not be present in anonymous maps
     expect(owner_session.locator(".umap-field-owner")).to_be_hidden()
 
 
@@ -135,15 +140,15 @@ def test_can_change_perms_after_create(tilelayer, live_server, page):
     page.get_by_title("Manage layers").click()
     page.get_by_title("Add a layer").click()
     page.locator("input[name=name]").fill("Layer 1")
-    save = page.get_by_role("button", name="Save")
-    expect(save).to_be_visible()
+    expect(
+        page.get_by_role("button", name="Visibility: Draft (private)")
+    ).to_be_visible()
+    expect(page.get_by_role("button", name="Save", exact=True)).to_be_hidden()
     with page.expect_response(re.compile(r".*/datalayer/create/.*")):
-        save.click()
+        page.get_by_role("button", name="Save draft", exact=True).click()
     edit_permissions = page.get_by_title("Update permissions and editors")
     expect(edit_permissions).to_be_visible()
     edit_permissions.click()
-    select = page.locator(".umap-field-share_status select")
-    expect(select).to_be_hidden()
     owner_field = page.locator(".umap-field-owner")
     expect(owner_field).to_be_hidden()
     editors_field = page.locator(".umap-field-editors input")
@@ -157,6 +162,9 @@ def test_can_change_perms_after_create(tilelayer, live_server, page):
     )
     expect(option).to_have_text("Inherit")
     expect(page.get_by_label("Secret edit link:")).to_be_visible()
+    page.locator('select[name="share_status"]').select_option("1")
+    expect(page.get_by_role("button", name="Save draft", exact=True)).to_be_hidden()
+    expect(page.get_by_role("button", name="Save", exact=True)).to_be_visible()
 
 
 def test_alert_message_after_create(
@@ -232,7 +240,7 @@ def test_anonymous_owner_can_delete_the_map(anonymap, live_server, owner_session
     owner_session.get_by_role("button", name="Delete").click()
     with owner_session.expect_response(re.compile(r".*/update/delete/.*")):
         owner_session.get_by_role("button", name="OK").click()
-    assert not Map.objects.count()
+    assert Map.objects.get(pk=anonymap.pk).share_status == Map.DELETED
 
 
 def test_non_owner_cannot_see_delete_button(anonymap, live_server, page):

umap/tests/integration/test_browser.py

@@ -15,7 +15,12 @@ DATALAYER_DATA = {
     "features": [
         {
             "type": "Feature",
-            "properties": {"name": "one point in france", "foo": "point", "bar": "one"},
+            "properties": {
+                "name": "one point in france",
+                "foo": "point",
+                "bar": "one",
+                "label": "this is label one",
+            },
             "geometry": {"type": "Point", "coordinates": [3.339844, 46.920255]},
         },
         {
@@ -24,6 +29,7 @@ DATALAYER_DATA = {
                 "name": "one polygon in greenland",
                 "foo": "polygon",
                 "bar": "two",
+                "label": "this is label two",
             },
             "geometry": {
                 "type": "Polygon",
@@ -44,6 +50,7 @@ DATALAYER_DATA = {
                 "name": "one line in new zeland",
                 "foo": "line",
                 "bar": "three",
+                "label": "this is label three",
             },
             "geometry": {
                 "type": "LineString",
@@ -476,3 +483,11 @@ def test_main_toolbox_toggle_all_layers(live_server, map, page):
     # Should hidden again all layers
     expect(page.locator(".datalayer.off")).to_have_count(3)
     expect(markers).to_have_count(0)
+
+
+def test_honour_the_label_fields_settings(live_server, map, page, bootstrap, settings):
+    settings.UMAP_LABEL_KEYS = ["label", "name"]
+    page.goto(f"{live_server.url}{map.get_absolute_url()}")
+    expect(page.locator(".panel").get_by_text("this is label one")).to_be_visible()
+    expect(page.locator(".panel").get_by_text("this is label two")).to_be_visible()
+    expect(page.locator(".panel").get_by_text("this is label three")).to_be_visible()

umap/tests/integration/test_dashboard.py

@@ -22,7 +22,7 @@ def test_owner_can_delete_map_after_confirmation(map, live_server, login):
     with page.expect_navigation():
         delete_button.click()
     assert dialog_shown
-    assert Map.objects.all().count() == 0
+    assert Map.objects.get(pk=map.pk).share_status == Map.DELETED
 
 
 def test_dashboard_map_preview(map, live_server, datalayer, login):

umap/tests/integration/test_edit_datalayer.py

@@ -120,7 +120,7 @@ def test_can_change_name(live_server, openmap, page, datalayer):
     page.locator('input[name="name"]').press("Control+a")
     page.locator('input[name="name"]').fill("new name")
     expect(page.locator(".umap-browser .datalayer")).to_contain_text("new name")
-    expect(page.locator(".umap-is-dirty")).to_be_visible()
+    expect(page.locator("body")).to_have_class(re.compile(".*umap-is-dirty.*"))
     with page.expect_response(re.compile(".*/datalayer/update/.*")):
         page.get_by_role("button", name="Save").click()
     saved = DataLayer.objects.last()
@@ -129,32 +129,43 @@ def test_can_change_name(live_server, openmap, page, datalayer):
 
 
 def test_can_create_new_datalayer(live_server, openmap, page, datalayer):
+    """
+    Test the creation and editing of a new datalayer.
+
+    This test verifies that:
+    1. A new datalayer can be created and saved.
+    2. The newly created datalayer appears in the UI and is saved in the database.
+    3. Editing the same datalayer updates it instead of creating a new one.
+    4. The UI reflects the changes and the database is updated correctly.
+    5. The 'dirty' state of the map is managed correctly during these operations.
+    """
+
     page.goto(
         f"{live_server.url}{openmap.get_absolute_url()}?edit&onLoadPanel=databrowser"
     )
     page.get_by_role("link", name="Manage layers").click()
     page.get_by_role("button", name="Add a layer").click()
     page.locator('input[name="name"]').click()
-    page.locator('input[name="name"]').fill("my new layer")
-    expect(page.get_by_text("my new layer")).to_be_visible()
+    page.locator('input[name="name"]').fill("Layer A")
+    expect(page.get_by_text("Layer A")).to_be_visible()
     with page.expect_response(re.compile(".*/datalayer/create/.*")):
         page.get_by_role("button", name="Save").click()
     assert DataLayer.objects.count() == 2
     saved = DataLayer.objects.last()
-    assert saved.name == "my new layer"
+    assert saved.name == "Layer A"
     expect(page.locator(".umap-is-dirty")).to_be_hidden()
     # Edit again, it should not create a new datalayer
     page.get_by_role("link", name="Manage layers").click()
     page.locator(".panel.right").get_by_title("Edit", exact=True).first.click()
     page.locator('input[name="name"]').click()
-    page.locator('input[name="name"]').fill("my new layer with a new name")
-    expect(page.get_by_text("my new layer with a new name")).to_be_visible()
+    page.locator('input[name="name"]').fill("Layer A with a new name")
+    expect(page.get_by_text("Layer A with a new name")).to_be_visible()
     page.get_by_role("button", name="Save").click()
     with page.expect_response(re.compile(".*/datalayer/update/.*")):
         page.get_by_role("button", name="Save").click()
     assert DataLayer.objects.count() == 2
     saved = DataLayer.objects.last()
-    assert saved.name == "my new layer with a new name"
+    assert saved.name == "Layer A with a new name"
     expect(page.locator(".umap-is-dirty")).to_be_hidden()
 
 

umap/tests/integration/test_import.py

@@ -71,9 +71,8 @@ def test_umap_import_from_file(live_server, tilelayer, page):
     expect(nonloaded).to_have_count(1)
 
 
+@pytest.mark.skip
 def test_umap_import_from_textarea(live_server, tilelayer, page, settings):
-    page.route("https://tile.openstreetmap.fr/hot/**", mock_tiles)
-
     settings.UMAP_ALLOW_ANONYMOUS = True
     page.goto(f"{live_server.url}/map/new/")
     page.get_by_role("button", name="Open browser").click()
@@ -94,11 +93,11 @@ def test_umap_import_from_textarea(live_server, tilelayer, page, settings):
     expect(
         page.locator('img[src="https://tile.openstreetmap.fr/hot/6/32/21.png"]')
     ).to_be_visible()
-    # Should not have imported umap_id, while in the file options
-    assert not page.evaluate("U.MAP.options.umap_id")
+    # Should not have imported id, while in the file options
+    assert not page.evaluate("U.MAP.properties.id")
     with page.expect_response(re.compile(r".*/datalayer/create/.*")):
         page.get_by_role("button", name="Save").click()
-    assert page.evaluate("U.MAP.options.umap_id")
+    assert page.evaluate("U.MAP.properties.id")
 
 
 def test_import_geojson_from_textarea(tilelayer, live_server, page):
@@ -607,6 +606,7 @@ def test_overpass_import_with_bbox(page, live_server, tilelayer, settings):
     }
     page.goto(f"{live_server.url}/map/new/")
     page.get_by_role("link", name="Import data").click()
+    page.get_by_role("button", name="Import helpers").click()
     page.get_by_role("button", name="Overpass").click()
     page.get_by_placeholder("amenity=drinking_water").fill("building")
     page.get_by_role("button", name="Choose this data").click()
@@ -657,6 +657,7 @@ def test_overpass_import_retains_boundary(page, live_server, tilelayer, settings
     page.route(re.compile("https://foobar.io/api.*"), handle)
     page.goto(f"{live_server.url}/map/new/")
     page.get_by_role("link", name="Import data").click()
+    page.get_by_role("button", name="Import helpers").click()
     page.get_by_role("button", name="Overpass").click()
     page.get_by_placeholder("amenity=drinking_water").fill("building")
     page.get_by_placeholder("Type area name, or let empty").click()
@@ -669,6 +670,7 @@ def test_overpass_import_retains_boundary(page, live_server, tilelayer, settings
     expect(page.get_by_placeholder("Provide an URL here")).to_have_value(
         "https://my.overpass.io/interpreter?data=[out:json];nwr[building](area:3601393025);out geom;"
     )
+    page.get_by_role("button", name="Import helpers").click()
     page.get_by_role("button", name="Overpass").click()
     expect(page.locator("#area")).to_contain_text(
         "Bray-sur-Seine, Seine-et-Marne, Île-de-France, France"
@@ -710,6 +712,7 @@ def test_import_from_datasets(page, live_server, tilelayer, settings):
     page.goto(f"{live_server.url}/map/new/")
     expect(page.locator(".leaflet-marker-icon")).to_be_hidden()
     page.get_by_role("link", name="Import data").click()
+    page.get_by_role("button", name="Import helpers").click()
     page.get_by_role("button", name="Datasets").click()
     page.get_by_role("dialog").get_by_role("combobox").select_option(
         "https://remote.org/data.json"