umap-project 2.8.0a0__py3-none-any.whl → 2.8.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of umap-project might be problematic. Click here for more details.

umap/__init__.py CHANGED
@@ -1 +1 @@
1
- VERSION = "2.8.0a0"
1
+ VERSION = "2.8.0a1"
@@ -2,7 +2,7 @@ from django.conf import settings
2
2
  from django.core.management.base import BaseCommand
3
3
 
4
4
  from umap.models import DataLayer
5
- from umap.storage import UmapFileSystem
5
+ from umap.storage.fs import FSDataStorage
6
6
 
7
7
 
8
8
  class Command(BaseCommand):
@@ -11,9 +11,9 @@ class Command(BaseCommand):
11
11
  def handle(self, *args, **options):
12
12
  assert settings.UMAP_READONLY, "You must run that script with a read-only uMap."
13
13
  assert (
14
- settings.STORAGES["data"]["BACKEND"] == "umap.storage.UmapS3"
14
+ settings.STORAGES["data"]["BACKEND"] == "umap.storage.s3.S3DataStorage"
15
15
  ), "You must configure your storages to point to S3"
16
- fs_storage = UmapFileSystem()
16
+ fs_storage = FSDataStorage()
17
17
  for datalayer in DataLayer.objects.all():
18
18
  geojson_fs_path = str(datalayer.geojson)
19
19
  try:
umap/settings/base.py CHANGED
@@ -176,10 +176,10 @@ STORAGES = {
176
176
  "BACKEND": "django.core.files.storage.FileSystemStorage",
177
177
  },
178
178
  "data": {
179
- "BACKEND": "umap.storage.UmapFileSystem",
179
+ "BACKEND": "umap.storage.fs.FSDataStorage",
180
180
  },
181
181
  "staticfiles": {
182
- "BACKEND": "umap.storage.UmapManifestStaticFilesStorage",
182
+ "BACKEND": "umap.storage.staticfiles.UmapManifestStaticFilesStorage",
183
183
  },
184
184
  }
185
185
  # Add application/json and application/geo+json to default django-storages setting
@@ -0,0 +1,3 @@
1
+ # Retrocompat
2
+
3
+ from .staticfiles import UmapManifestStaticFilesStorage # noqa: F401
umap/storage/fs.py ADDED
@@ -0,0 +1,101 @@
1
+ import operator
2
+ import os
3
+ import time
4
+ from pathlib import Path
5
+
6
+ from django.conf import settings
7
+ from django.core.files.storage import FileSystemStorage
8
+
9
+
10
+ class FSDataStorage(FileSystemStorage):
11
+ def get_reference_version(self, instance):
12
+ return self._extract_version_ref(instance.geojson.name)
13
+
14
+ def make_filename(self, instance):
15
+ root = self._base_path(instance)
16
+ name = "%s_%s.geojson" % (instance.pk, int(time.time() * 1000))
17
+ return root / name
18
+
19
+ def list_versions(self, instance):
20
+ root = self._base_path(instance)
21
+ names = self.listdir(root)[1]
22
+ names = [name for name in names if self._is_valid_version(name, instance)]
23
+ versions = [self._version_metadata(name, instance) for name in names]
24
+ versions.sort(reverse=True, key=operator.itemgetter("at"))
25
+ return versions
26
+
27
+ def get_version(self, ref, instance):
28
+ with self.open(self.get_version_path(ref, instance), "r") as f:
29
+ return f.read()
30
+
31
+ def get_version_path(self, ref, instance):
32
+ base_path = Path(settings.MEDIA_ROOT) / self._base_path(instance)
33
+ fullpath = base_path / f"{instance.pk}_{ref}.geojson"
34
+ if instance.old_id and not fullpath.exists():
35
+ fullpath = base_path / f"{instance.old_id}_{ref}.geojson"
36
+ if not fullpath.exists():
37
+ raise ValueError(f"Invalid version reference: {ref}")
38
+ return fullpath
39
+
40
+ def onDatalayerSave(self, instance):
41
+ self._purge_gzip(instance)
42
+ self._purge_old_versions(instance, keep=settings.UMAP_KEEP_VERSIONS)
43
+
44
+ def onDatalayerDelete(self, instance):
45
+ self._purge_gzip(instance)
46
+ self._purge_old_versions(instance, keep=None)
47
+
48
+ def _extract_version_ref(self, path):
49
+ version = path.split(".")[0]
50
+ if "_" in version:
51
+ return version.split("_")[-1]
52
+ return version
53
+
54
+ def _base_path(self, instance):
55
+ path = ["datalayer", str(instance.map.pk)[-1]]
56
+ if len(str(instance.map.pk)) > 1:
57
+ path.append(str(instance.map.pk)[-2])
58
+ path.append(str(instance.map.pk))
59
+ return Path(os.path.join(*path))
60
+
61
+ def _is_valid_version(self, name, instance):
62
+ valid_prefixes = [name.startswith("%s_" % instance.pk)]
63
+ if instance.old_id:
64
+ valid_prefixes.append(name.startswith("%s_" % instance.old_id))
65
+ return any(valid_prefixes) and name.endswith(".geojson")
66
+
67
+ def _version_metadata(self, name, instance):
68
+ ref = self._extract_version_ref(name)
69
+ return {
70
+ "name": name,
71
+ "ref": ref,
72
+ "at": ref,
73
+ "size": self.size(self._base_path(instance) / name),
74
+ }
75
+
76
+ def _purge_old_versions(self, instance, keep=None):
77
+ root = self._base_path(instance)
78
+ versions = self.list_versions(instance)
79
+ if keep is not None:
80
+ versions = versions[keep:]
81
+ for version in versions:
82
+ name = version["name"]
83
+ # Should not be in the list, but ensure to not delete the file
84
+ # currently used in database
85
+ if keep is not None and instance.geojson.name.endswith(name):
86
+ continue
87
+ try:
88
+ self.delete(root / name)
89
+ except FileNotFoundError:
90
+ pass
91
+
92
+ def _purge_gzip(self, instance):
93
+ root = self._base_path(instance)
94
+ names = self.listdir(root)[1]
95
+ prefixes = [f"{instance.pk}_"]
96
+ if instance.old_id:
97
+ prefixes.append(f"{instance.old_id}_")
98
+ prefixes = tuple(prefixes)
99
+ for name in names:
100
+ if name.startswith(prefixes) and name.endswith(".gz"):
101
+ self.delete(root / name)
umap/storage/s3.py ADDED
@@ -0,0 +1,61 @@
1
+ from gzip import GzipFile
2
+
3
+ from django.core.exceptions import ImproperlyConfigured
4
+
5
+ try:
6
+ from botocore.exceptions import ClientError
7
+ from storages.backends.s3 import S3Storage
8
+ except ImportError:
9
+ raise ImproperlyConfigured(
10
+ "You need to install s3 dependencies: pip install umap-project[s3]"
11
+ )
12
+
13
+
14
+ class S3DataStorage(S3Storage):
15
+ gzip = True
16
+
17
+ def get_reference_version(self, instance):
18
+ metadata = self.connection.meta.client.head_object(
19
+ Bucket=self.bucket_name, Key=instance.geojson.name
20
+ )
21
+ # Do not fail if bucket does not handle versioning
22
+ return metadata.get("VersionId", metadata["ETag"])
23
+
24
+ def make_filename(self, instance):
25
+ return f"{str(instance.pk)}.geojson"
26
+
27
+ def list_versions(self, instance):
28
+ response = self.connection.meta.client.list_object_versions(
29
+ Bucket=self.bucket_name, Prefix=instance.geojson.name
30
+ )
31
+ return [
32
+ {
33
+ "ref": version["VersionId"],
34
+ "at": version["LastModified"].timestamp() * 1000,
35
+ "size": version["Size"],
36
+ }
37
+ for version in response["Versions"]
38
+ ]
39
+
40
+ def get_version(self, ref, instance):
41
+ try:
42
+ data = self.connection.meta.client.get_object(
43
+ Bucket=self.bucket_name,
44
+ Key=instance.geojson.name,
45
+ VersionId=ref,
46
+ )
47
+ except ClientError:
48
+ raise ValueError(f"Invalid version reference: {ref}")
49
+ return GzipFile(mode="r", fileobj=data["Body"]).read()
50
+
51
+ def get_version_path(self, ref, instance):
52
+ return self.url(instance.geojson.name, parameters={"VersionId": ref})
53
+
54
+ def onDatalayerSave(self, instance):
55
+ pass
56
+
57
+ def onDatalayerDelete(self, instance):
58
+ return self.connection.meta.client.delete_object(
59
+ Bucket=self.bucket_name,
60
+ Key=instance.geojson.name,
61
+ )
@@ -0,0 +1,64 @@
1
+ from pathlib import Path
2
+
3
+ from django.conf import settings
4
+ from django.contrib.staticfiles.storage import ManifestStaticFilesStorage
5
+ from rcssmin import cssmin
6
+ from rjsmin import jsmin
7
+
8
+
9
+ class UmapManifestStaticFilesStorage(ManifestStaticFilesStorage):
10
+ support_js_module_import_aggregation = True
11
+ max_post_process_passes = 15
12
+
13
+ # We remove `;` at the end of all regexps to match our biome config.
14
+ _js_module_import_aggregation_patterns = (
15
+ "*.js",
16
+ (
17
+ (
18
+ (
19
+ r"""(?P<matched>import(?s:(?P<import>[\s\{].*?))"""
20
+ r"""\s*from\s*['"](?P<url>[\.\/].*?)["']\s*)"""
21
+ ),
22
+ 'import%(import)s from "%(url)s"\n',
23
+ ),
24
+ (
25
+ (
26
+ r"""(?P<matched>export(?s:(?P<exports>[\s\{].*?))"""
27
+ r"""\s*from\s*["'](?P<url>[\.\/].*?)["']\s*)"""
28
+ ),
29
+ 'export%(exports)s from "%(url)s"\n',
30
+ ),
31
+ (
32
+ r"""(?P<matched>import\s*['"](?P<url>[\.\/].*?)["']\s*)""",
33
+ 'import"%(url)s"\n',
34
+ ),
35
+ (
36
+ r"""(?P<matched>import\(["'](?P<url>.*?)["']\)\.then)""",
37
+ """import("%(url)s").then""",
38
+ ),
39
+ (
40
+ r"""(?P<matched>await import\(["'](?P<url>.*?)["']\))""",
41
+ """await import("%(url)s")""",
42
+ ),
43
+ ),
44
+ )
45
+
46
+ def post_process(self, paths, **options):
47
+ collected = super().post_process(paths, **options)
48
+ for original_path, processed_path, processed in collected:
49
+ if isinstance(processed, Exception):
50
+ print("Error with file", original_path)
51
+ raise processed
52
+ if processed_path.endswith(".js"):
53
+ path = Path(settings.STATIC_ROOT) / processed_path
54
+ initial = path.read_text()
55
+ if "sourceMappingURL" not in initial: # Already minified.
56
+ minified = jsmin(initial)
57
+ path.write_text(minified)
58
+ if processed_path.endswith(".css"):
59
+ path = Path(settings.STATIC_ROOT) / processed_path
60
+ initial = path.read_text()
61
+ if "sourceMappingURL" not in initial: # Already minified.
62
+ minified = cssmin(initial)
63
+ path.write_text(minified)
64
+ yield original_path, processed_path, True
@@ -27,7 +27,7 @@ def patch_storage():
27
27
 
28
28
  DataLayer.geojson.field.storage = storages.create_storage(
29
29
  {
30
- "BACKEND": "umap.storage.UmapS3",
30
+ "BACKEND": "umap.storage.s3.S3DataStorage",
31
31
  "OPTIONS": {
32
32
  "access_key": "testing",
33
33
  "secret_key": "testing",
@@ -15,7 +15,7 @@ def staticfiles(settings):
15
15
  # Make sure settings are properly reset after the test
16
16
  settings.STORAGES = deepcopy(settings.STORAGES)
17
17
  settings.STORAGES["staticfiles"]["BACKEND"] = (
18
- "umap.storage.UmapManifestStaticFilesStorage"
18
+ "umap.storage.staticfiles.UmapManifestStaticFilesStorage"
19
19
  )
20
20
  try:
21
21
  call_command("collectstatic", "--noinput")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: umap-project
3
- Version: 2.8.0a0
3
+ Version: 2.8.0a1
4
4
  Summary: Create maps with OpenStreetMap layers in a minute and embed them in your site.
5
5
  Author-email: Yohan Boniface <yb@enix.org>
6
6
  Maintainer-email: David Larlet <david@larlet.fr>
@@ -1,4 +1,4 @@
1
- umap/__init__.py,sha256=2KCwAhPAdl9iGZ4OqB3b9x9oEMjwYBXpF7OB99CO7m8,20
1
+ umap/__init__.py,sha256=GQ1xPGwR_--i3-Pg0nkWcPAdD7uLAEBfX3aVZiWlVc0,20
2
2
  umap/admin.py,sha256=LoQytPGK6pLBqZ5QgQ9DIPAxhTG31cTtHOCqO9BY5S4,2645
3
3
  umap/apps.py,sha256=5ssKqPUuNJlapaBmr4LY_HDb7J1NFCT3wzythxQOOfs,109
4
4
  umap/asgi.py,sha256=CuVSNBwNb4AvuaD_Ha3ehtvf-c46ijZoVOSoP6WhXp8,432
@@ -10,7 +10,6 @@ umap/forms.py,sha256=fonoSwA02LawR7kXbjEZCH0ZYi53fAbRHYgW2RaqeYw,3803
10
10
  umap/managers.py,sha256=-lBK0xYFRDfX76qDRdLnZOA8jEPYseEwIj8QOiHVM4w,243
11
11
  umap/middleware.py,sha256=p8EPW_gYW8Wh2lk0DNIAkZQbYlBZugW7Yq4iiA7L4aE,514
12
12
  umap/models.py,sha256=4SzhKdyWXfJMdzEpCyVPnzbTT-TGbDusAy7SP_8UuwI,17929
13
- umap/storage.py,sha256=kEzS0BP9jrfVwlUtmEcf4W0t-7uEBqFfgKBlKvQTRVg,7900
14
13
  umap/urls.py,sha256=LA3zxyu-GDo8kVqdyU7_bdbDGhDJV8_yFW4oEPTXw4s,7559
15
14
  umap/utils.py,sha256=19i8ibi-1IXxafT4k_yOHMhD-DsPH74Ll9qw-UrUkM4,5856
16
15
  umap/views.py,sha256=xSKBdbRFtoA0RXm5qCrVAoir4gm0_rN7d-70Nzbg0pQ,46016
@@ -118,7 +117,7 @@ umap/management/commands/clean_tilelayer.py,sha256=Rcc2PibUUreU0jUZMtUlyqVvgbQML
118
117
  umap/management/commands/empty_trash.py,sha256=guGnu72EYOiRtS-pLjxByFOmFtq1Ra0JR0SWBOZDOsY,1024
119
118
  umap/management/commands/generate_js_locale.py,sha256=wkf-PFIHS7m4ZhyL1ZRMBLqyUeY2SlOrTXS42tE0-bs,1281
120
119
  umap/management/commands/import_pictograms.py,sha256=RuQDCoiKamba4l3fZUGAXRyd-3zwWWT5c5AhgDvs7AQ,2369
121
- umap/management/commands/migrate_to_S3.py,sha256=-LVbLdr554mjhdSty1OE_33OI4CjdegqQZr2Fw2JeSk,1119
120
+ umap/management/commands/migrate_to_S3.py,sha256=GBGnydc107v75NYsQfMLLO7Jx0i2g7EKEfE00YZVb1M,1130
122
121
  umap/management/commands/run_websocket_server.py,sha256=TyECJWnmZ95KpVEWSaqfXywz5VwIEzPdypU2d6V541c,648
123
122
  umap/migrations/0001_initial.py,sha256=dMcXtTKPiA0IqXCrDVctH91Fe0hhc04NxmvcLAULyzE,8787
124
123
  umap/migrations/0002_tilelayer_tms.py,sha256=E99JAu1K0NzwsCEJs1z5uGlBkBJmoVb9a3WBKjpLYlo,372
@@ -147,7 +146,7 @@ umap/migrations/0024_alter_map_share_status.py,sha256=QRjERy2XN0jsc8MM5cCba5freq
147
146
  umap/migrations/0025_alter_datalayer_geojson.py,sha256=958v9AkpkAR5Q78ZcHC0fgZzN05BdfQwtNvUuPmWEvI,563
148
147
  umap/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
149
148
  umap/settings/__init__.py,sha256=aPJkOTk0uFusIBA-uOjdUi10R5Cxt4jl5yv2_uCTUvo,1702
150
- umap/settings/base.py,sha256=zdJa2Qqq1hlsFh80YFhOiNEs1cWa9ZAE5zzygpmHgKE,11082
149
+ umap/settings/base.py,sha256=rhaIDby2wSb4v8IBx_6xqHVnIigD4G0xzdHX2-9UfNM,11096
151
150
  umap/settings/dev.py,sha256=pj1mpmZXiI2syW8pB01wcVeqCFABF3V-nlOxArir4cw,386
152
151
  umap/settings/local.py.sample,sha256=wpnoe7qtXer_xBuhWbcbqcSCotTJRu6h8hG7N-sD0b4,3157
153
152
  umap/static/.gitignore,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -449,6 +448,10 @@ umap/static/umap/vendors/tokml/tokml.es.js,sha256=BqL0WqFH5UZAh_S_265E6PWZjPMYxe
449
448
  umap/static/umap/vendors/tokml/tokml.es.mjs.map,sha256=vw5JxZFh_2_xM1cHI51r1Bf48JaBKzNcR7ddgaaF8KY,45844
450
449
  umap/static/umap/vendors/toolbar/leaflet.toolbar.css,sha256=5KVBOQ0ivsFuafKYvVm32wJ_fi7w8Li1-2Rwwcv85jA,2244
451
450
  umap/static/umap/vendors/toolbar/leaflet.toolbar.js,sha256=HXh_bR49ZFpJ-GNXDNo2eHy-fJmrWehAzUeHgGhu__c,5326
451
+ umap/storage/__init__.py,sha256=Aj421eIsZhsu0B7zd5lTJufVYr0EtUkH0lTqpbBh8AU,85
452
+ umap/storage/fs.py,sha256=CdjnDe4UVMt_9Yp2uZqvtarfHTAYRed6Tid2UU-sBgY,3721
453
+ umap/storage/s3.py,sha256=KAYu3vAqXbd5UhaoPxG6zcGtBfKZOzzi-6uY6YEuIcY,1962
454
+ umap/storage/staticfiles.py,sha256=mxFcenC1JECmpNy4H0e7vX8GObDZVXzs1RPjQFWNd5k,2473
452
455
  umap/templates/404.html,sha256=1yLlD8rSF_9cfjm5FYy--P46HLVbHeFkJiW9nRzM--E,399
453
456
  umap/templates/500.html,sha256=Z8x47OVfYXquAYAlmRB0EJVTCiCaBppFFiFEmoYsMYY,5202
454
457
  umap/templates/base.html,sha256=_Q0Ikwese3vlUl0pKQdzHDy_oRT9OV4uWh0RGFaYAQA,1499
@@ -495,14 +498,14 @@ umap/tests/conftest.py,sha256=KQCZanCTl1ABLIKOuyxS_cpBoXGiwjDc29jsLBiSWxY,1633
495
498
  umap/tests/settings.py,sha256=tY70LMFXyo_WijswqGyeWai7vBzM62k7IA8pkkbc9y4,816
496
499
  umap/tests/test_clean_tilelayer.py,sha256=wGTd_AHOTmQ4QMswAyc-1_lJmQOSyhY3OahLAusEIdA,2515
497
500
  umap/tests/test_datalayer.py,sha256=NWX7o-sLOrq3nHT0GDywz5vtJU4HJhZZh11r_rWxhVA,9539
498
- umap/tests/test_datalayer_s3.py,sha256=bqLpdQZLoL-jwD7HeqMXuLFUfnkP-xd6B6BBW7JFm0U,4145
501
+ umap/tests/test_datalayer_s3.py,sha256=6V3AK22AXkFNjx5__SyBk0Uj0rTDAjJQv67r7D_MDVc,4155
499
502
  umap/tests/test_datalayer_views.py,sha256=Fx_oQF3hBC2FVHTTjTScXbFS2d7FRKdBL7QFFexvKkg,22880
500
503
  umap/tests/test_empty_trash.py,sha256=9dYdnQqzlfgkExQxiQDqQ4flKsPTZ97uB3te5wmZ0Nw,1112
501
504
  umap/tests/test_licence.py,sha256=BxNY3gdKhIoc2u5OPmAkmjCp0jJN-Jm-uPOfAZlpOHA,339
502
505
  umap/tests/test_map.py,sha256=vrtheSMQNk45kBIcJ0QY9K7HKYee5yg4Vnp78DyaIwQ,5170
503
506
  umap/tests/test_map_views.py,sha256=EKLJnQ-xk_bkaJ6P7OJ2bnya5orbaSnWFV8GIYcjDNk,33823
504
507
  umap/tests/test_merge_features.py,sha256=uLZSW00WAI8_nZS0KPP8gg8U4nnky-XGb-VhhKUxv1M,2275
505
- umap/tests/test_statics.py,sha256=WJe4DZ-cSfN_wCRD8U9ocl6v5FoXrVwBjU6kI6BOcmY,1252
508
+ umap/tests/test_statics.py,sha256=xKuxT8Xj5Ii7gKISuiSfDj7dpjmJ2Ierby3Lg-haZCg,1264
506
509
  umap/tests/test_team_views.py,sha256=vExhJ3c1cJ7vgxe0G20UzTKkzR5D2UgAapk09muUg5w,4481
507
510
  umap/tests/test_tilelayer.py,sha256=toVpVutEvMLWKx5uH7ZbGNPGzqICZx1_S2OOpIfYPfQ,603
508
511
  umap/tests/test_utils.py,sha256=noh-AFL3qV-dNZYr8L1acsYC02SI710Bq2ZXV-jBEzk,407
@@ -569,8 +572,8 @@ umap/tests/integration/test_view_marker.py,sha256=ZLS6-GOWYpjeoYGHiHa7HesXJTLu9w
569
572
  umap/tests/integration/test_view_polygon.py,sha256=NMJC6Nt9VpQ8FIU9Pqq2OspHv49xsWlsoXCr8iBa0VA,2060
570
573
  umap/tests/integration/test_view_polyline.py,sha256=aJoXKmLhJaN0yhPdDCVskZNGx3q3mLDkjVPhZ30cadA,13959
571
574
  umap/tests/integration/test_websocket_sync.py,sha256=Xjn8z7Gj2PAmPmLkMTsHztFmhzsfyE3vg-wfewpA2I4,15511
572
- umap_project-2.8.0a0.dist-info/METADATA,sha256=a0AHRG8dpF_7KYUQNFAGMe84KPD56yidRdzGIilsJCU,2993
573
- umap_project-2.8.0a0.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
574
- umap_project-2.8.0a0.dist-info/entry_points.txt,sha256=gz-KDQfEsMLBae8ABOD3foJsCYGPW1tA4Y394R_1RW8,39
575
- umap_project-2.8.0a0.dist-info/licenses/LICENSE,sha256=kQtrtRKgiPhcl7aO0-lmvbrNAXu7WHyiXvPrUk-TD2Q,820
576
- umap_project-2.8.0a0.dist-info/RECORD,,
575
+ umap_project-2.8.0a1.dist-info/METADATA,sha256=MRv8Aho0zddPOihVvrm8zDHu-JYcjSARVr8ZlsO8Shg,2993
576
+ umap_project-2.8.0a1.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
577
+ umap_project-2.8.0a1.dist-info/entry_points.txt,sha256=gz-KDQfEsMLBae8ABOD3foJsCYGPW1tA4Y394R_1RW8,39
578
+ umap_project-2.8.0a1.dist-info/licenses/LICENSE,sha256=kQtrtRKgiPhcl7aO0-lmvbrNAXu7WHyiXvPrUk-TD2Q,820
579
+ umap_project-2.8.0a1.dist-info/RECORD,,
umap/storage.py DELETED
@@ -1,216 +0,0 @@
1
- import operator
2
- import os
3
- import shutil
4
- import time
5
- from gzip import GzipFile
6
- from pathlib import Path
7
-
8
- from botocore.exceptions import ClientError
9
- from django.conf import settings
10
- from django.contrib.staticfiles.storage import ManifestStaticFilesStorage
11
- from django.core.files.storage import FileSystemStorage
12
- from rcssmin import cssmin
13
- from rjsmin import jsmin
14
- from storages.backends.s3 import S3Storage
15
-
16
-
17
- class UmapManifestStaticFilesStorage(ManifestStaticFilesStorage):
18
- support_js_module_import_aggregation = True
19
- max_post_process_passes = 15
20
-
21
- # We remove `;` at the end of all regexps to match our biome config.
22
- _js_module_import_aggregation_patterns = (
23
- "*.js",
24
- (
25
- (
26
- (
27
- r"""(?P<matched>import(?s:(?P<import>[\s\{].*?))"""
28
- r"""\s*from\s*['"](?P<url>[\.\/].*?)["']\s*)"""
29
- ),
30
- 'import%(import)s from "%(url)s"\n',
31
- ),
32
- (
33
- (
34
- r"""(?P<matched>export(?s:(?P<exports>[\s\{].*?))"""
35
- r"""\s*from\s*["'](?P<url>[\.\/].*?)["']\s*)"""
36
- ),
37
- 'export%(exports)s from "%(url)s"\n',
38
- ),
39
- (
40
- r"""(?P<matched>import\s*['"](?P<url>[\.\/].*?)["']\s*)""",
41
- 'import"%(url)s"\n',
42
- ),
43
- (
44
- r"""(?P<matched>import\(["'](?P<url>.*?)["']\)\.then)""",
45
- """import("%(url)s").then""",
46
- ),
47
- (
48
- r"""(?P<matched>await import\(["'](?P<url>.*?)["']\))""",
49
- """await import("%(url)s")""",
50
- ),
51
- ),
52
- )
53
-
54
- def post_process(self, paths, **options):
55
- collected = super().post_process(paths, **options)
56
- for original_path, processed_path, processed in collected:
57
- if isinstance(processed, Exception):
58
- print("Error with file", original_path)
59
- raise processed
60
- if processed_path.endswith(".js"):
61
- path = Path(settings.STATIC_ROOT) / processed_path
62
- initial = path.read_text()
63
- if "sourceMappingURL" not in initial: # Already minified.
64
- minified = jsmin(initial)
65
- path.write_text(minified)
66
- if processed_path.endswith(".css"):
67
- path = Path(settings.STATIC_ROOT) / processed_path
68
- initial = path.read_text()
69
- if "sourceMappingURL" not in initial: # Already minified.
70
- minified = cssmin(initial)
71
- path.write_text(minified)
72
- yield original_path, processed_path, True
73
-
74
-
75
- class UmapS3(S3Storage):
76
- gzip = True
77
-
78
- def get_reference_version(self, instance):
79
- metadata = self.connection.meta.client.head_object(
80
- Bucket=self.bucket_name, Key=instance.geojson.name
81
- )
82
- # Do not fail if bucket does not handle versioning
83
- return metadata.get("VersionId", metadata["ETag"])
84
-
85
- def make_filename(self, instance):
86
- return f"{str(instance.pk)}.geojson"
87
-
88
- def list_versions(self, instance):
89
- response = self.connection.meta.client.list_object_versions(
90
- Bucket=self.bucket_name, Prefix=instance.geojson.name
91
- )
92
- return [
93
- {
94
- "ref": version["VersionId"],
95
- "at": version["LastModified"].timestamp() * 1000,
96
- "size": version["Size"],
97
- }
98
- for version in response["Versions"]
99
- ]
100
-
101
- def get_version(self, ref, instance):
102
- try:
103
- data = self.connection.meta.client.get_object(
104
- Bucket=self.bucket_name,
105
- Key=instance.geojson.name,
106
- VersionId=ref,
107
- )
108
- except ClientError:
109
- raise ValueError(f"Invalid version reference: {ref}")
110
- return GzipFile(mode="r", fileobj=data["Body"]).read()
111
-
112
- def get_version_path(self, ref, instance):
113
- return self.url(instance.geojson.name, parameters={"VersionId": ref})
114
-
115
- def onDatalayerSave(self, instance):
116
- pass
117
-
118
- def onDatalayerDelete(self, instance):
119
- return self.connection.meta.client.delete_object(
120
- Bucket=self.bucket_name,
121
- Key=instance.geojson.name,
122
- )
123
-
124
-
125
- class UmapFileSystem(FileSystemStorage):
126
- def get_reference_version(self, instance):
127
- return self._extract_version_ref(instance.geojson.name)
128
-
129
- def make_filename(self, instance):
130
- root = self._base_path(instance)
131
- name = "%s_%s.geojson" % (instance.pk, int(time.time() * 1000))
132
- return root / name
133
-
134
- def list_versions(self, instance):
135
- root = self._base_path(instance)
136
- names = self.listdir(root)[1]
137
- names = [name for name in names if self._is_valid_version(name, instance)]
138
- versions = [self._version_metadata(name, instance) for name in names]
139
- versions.sort(reverse=True, key=operator.itemgetter("at"))
140
- return versions
141
-
142
- def get_version(self, ref, instance):
143
- with self.open(self.get_version_path(ref, instance), "r") as f:
144
- return f.read()
145
-
146
- def get_version_path(self, ref, instance):
147
- base_path = Path(settings.MEDIA_ROOT) / self._base_path(instance)
148
- fullpath = base_path / f"{instance.pk}_{ref}.geojson"
149
- if instance.old_id and not fullpath.exists():
150
- fullpath = base_path / f"{instance.old_id}_{ref}.geojson"
151
- if not fullpath.exists():
152
- raise ValueError(f"Invalid version reference: {ref}")
153
- return fullpath
154
-
155
- def onDatalayerSave(self, instance):
156
- self._purge_gzip(instance)
157
- self._purge_old_versions(instance, keep=settings.UMAP_KEEP_VERSIONS)
158
-
159
- def onDatalayerDelete(self, instance):
160
- self._purge_gzip(instance)
161
- self._purge_old_versions(instance, keep=None)
162
-
163
- def _extract_version_ref(self, path):
164
- version = path.split(".")[0]
165
- if "_" in version:
166
- return version.split("_")[-1]
167
- return version
168
-
169
- def _base_path(self, instance):
170
- path = ["datalayer", str(instance.map.pk)[-1]]
171
- if len(str(instance.map.pk)) > 1:
172
- path.append(str(instance.map.pk)[-2])
173
- path.append(str(instance.map.pk))
174
- return Path(os.path.join(*path))
175
-
176
- def _is_valid_version(self, name, instance):
177
- valid_prefixes = [name.startswith("%s_" % instance.pk)]
178
- if instance.old_id:
179
- valid_prefixes.append(name.startswith("%s_" % instance.old_id))
180
- return any(valid_prefixes) and name.endswith(".geojson")
181
-
182
- def _version_metadata(self, name, instance):
183
- ref = self._extract_version_ref(name)
184
- return {
185
- "name": name,
186
- "ref": ref,
187
- "at": ref,
188
- "size": self.size(self._base_path(instance) / name),
189
- }
190
-
191
- def _purge_old_versions(self, instance, keep=None):
192
- root = self._base_path(instance)
193
- versions = self.list_versions(instance)
194
- if keep is not None:
195
- versions = versions[keep:]
196
- for version in versions:
197
- name = version["name"]
198
- # Should not be in the list, but ensure to not delete the file
199
- # currently used in database
200
- if keep is not None and instance.geojson.name.endswith(name):
201
- continue
202
- try:
203
- self.delete(root / name)
204
- except FileNotFoundError:
205
- pass
206
-
207
- def _purge_gzip(self, instance):
208
- root = self._base_path(instance)
209
- names = self.listdir(root)[1]
210
- prefixes = [f"{instance.pk}_"]
211
- if instance.old_id:
212
- prefixes.append(f"{instance.old_id}_")
213
- prefixes = tuple(prefixes)
214
- for name in names:
215
- if name.startswith(prefixes) and name.endswith(".gz"):
216
- self.delete(root / name)