igs-slm 0.1.4b0__py3-none-any.whl → 0.1.5b0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. igs_slm-0.1.5b0.dist-info/METADATA +115 -0
  2. {igs_slm-0.1.4b0.dist-info → igs_slm-0.1.5b0.dist-info}/RECORD +192 -172
  3. {igs_slm-0.1.4b0.dist-info → igs_slm-0.1.5b0.dist-info}/WHEEL +1 -1
  4. igs_slm-0.1.5b0.dist-info/entry_points.txt +3 -0
  5. {igs_slm-0.1.4b0.dist-info → igs_slm-0.1.5b0.dist-info/licenses}/LICENSE +1 -1
  6. slm/__init__.py +17 -14
  7. slm/admin.py +32 -5
  8. slm/api/edit/views.py +22 -9
  9. slm/api/public/views.py +9 -7
  10. slm/api/views.py +45 -6
  11. slm/apps.py +28 -6
  12. slm/authentication.py +3 -2
  13. slm/bin/startproject.py +102 -31
  14. slm/bin/templates/{{ project_dir }}/pyproject.toml +30 -21
  15. slm/bin/templates/{{ project_dir }}/{sites → src/sites}/{{ site }}/base.py +12 -1
  16. slm/bin/templates/{{ project_dir }}/{sites → src/sites}/{{ site }}/develop/__init__.py +5 -27
  17. slm/bin/templates/{{ project_dir }}/{sites → src/sites}/{{ site }}/production/__init__.py +6 -27
  18. slm/bin/templates/{{ project_dir }}/src/sites/{{ site }}/validation.py +29 -0
  19. slm/context.py +5 -0
  20. slm/defines/AlertLevel.py +10 -2
  21. slm/defines/AntennaCalibration.py +6 -4
  22. slm/defines/AntennaFeatures.py +12 -9
  23. slm/defines/AntennaReferencePoint.py +12 -10
  24. slm/defines/Aspiration.py +4 -2
  25. slm/defines/CardinalDirection.py +6 -4
  26. slm/defines/CollocationStatus.py +3 -1
  27. slm/defines/EquipmentState.py +6 -12
  28. slm/defines/FlagSeverity.py +4 -2
  29. slm/defines/FractureSpacing.py +7 -5
  30. slm/defines/FrequencyStandardType.py +6 -4
  31. slm/defines/GeodesyMLVersion.py +2 -0
  32. slm/defines/Instrumentation.py +9 -7
  33. slm/defines/LogEntryType.py +17 -15
  34. slm/defines/SLMFileType.py +4 -2
  35. slm/defines/SiteFileUploadStatus.py +7 -24
  36. slm/defines/SiteLogFormat.py +8 -32
  37. slm/defines/SiteLogStatus.py +9 -28
  38. slm/defines/TectonicPlates.py +18 -16
  39. slm/manage.py +24 -0
  40. slm/management/commands/check_upgrade.py +142 -0
  41. slm/management/commands/generate_sinex.py +110 -92
  42. slm/management/commands/head_from_index.py +11 -8
  43. slm/management/commands/import_archive.py +27 -18
  44. slm/management/commands/import_equipment.py +1 -1
  45. slm/management/commands/sitelog.py +1 -3
  46. slm/management/commands/synchronize.py +1 -1
  47. slm/management/commands/validate_db.py +6 -4
  48. slm/map/defines.py +18 -14
  49. slm/map/templates/slm/map.html +4 -6
  50. slm/migrations/0001_remove_archiveindex_no_overlapping_ranges_per_site_and_more.py +26 -0
  51. slm/migrations/0002_alter_archivedsitelog_file_and_more.py +44 -0
  52. slm/migrations/0003_alter_archivedsitelog_name_and_more.py +35 -0
  53. slm/migrations/0004_alter_site_name.py +24 -0
  54. slm/migrations/0005_slmversion.py +30 -0
  55. slm/migrations/0017_alter_logentry_unique_together_and_more.py +3 -1
  56. slm/migrations/0018_afix_deleted.py +3 -1
  57. slm/migrations/0018_alter_siteantenna_options_and_more.py +87 -56
  58. slm/migrations/0019_remove_siteantenna_marker_enu_siteantenna_marker_une_and_more.py +1 -1
  59. slm/migrations/0023_archivedsitelog_gml_version_and_more.py +1 -1
  60. slm/migrations/0031_alter_antenna_features.py +44 -0
  61. slm/migrations/0032_archiveindex_valid_range_and_more.py +84 -0
  62. slm/migrations/add_index_order_index.py +54 -0
  63. slm/migrations/normalize_index.py +147 -0
  64. slm/migrations/simplify_index.py +48 -0
  65. slm/migrations/verify_index.py +67 -0
  66. slm/models/__init__.py +2 -0
  67. slm/models/alerts.py +7 -10
  68. slm/models/data.py +1 -2
  69. slm/models/equipment.py +1 -1
  70. slm/models/fields.py +41 -0
  71. slm/models/index.py +183 -53
  72. slm/models/sitelog.py +35 -38
  73. slm/models/system.py +72 -31
  74. slm/models/user.py +1 -1
  75. slm/parsing/__init__.py +33 -16
  76. slm/parsing/legacy/binding.py +65 -34
  77. slm/parsing/legacy/parser.py +2 -2
  78. slm/parsing/xsd/binding.py +1 -1
  79. slm/parsing/xsd/parser.py +1 -2
  80. slm/receivers/__init__.py +2 -2
  81. slm/receivers/index.py +2 -1
  82. slm/receivers/migration.py +21 -0
  83. slm/settings/__init__.py +192 -4
  84. slm/settings/assets.py +26 -0
  85. slm/settings/auth.py +18 -14
  86. slm/settings/ckeditor.py +12 -6
  87. slm/settings/debug.py +2 -2
  88. slm/settings/emails.py +50 -0
  89. slm/settings/internationalization.py +8 -6
  90. slm/settings/logging.py +100 -88
  91. slm/settings/platform/darwin.py +16 -6
  92. slm/settings/rest.py +20 -15
  93. slm/settings/root.py +192 -98
  94. slm/settings/routines.py +5 -1
  95. slm/settings/secrets.py +20 -31
  96. slm/settings/security.py +7 -5
  97. slm/settings/slm.py +35 -16
  98. slm/settings/static_templates.py +12 -9
  99. slm/settings/templates.py +31 -25
  100. slm/settings/uploads.py +33 -5
  101. slm/settings/urls.py +1 -1
  102. slm/settings/validation.py +165 -165
  103. slm/signals.py +3 -2
  104. slm/static/slm/css/style.css +37 -36
  105. slm/static/slm/js/autocomplete.js +6 -4
  106. slm/static/slm/js/file_modal.js +62 -0
  107. slm/static/slm/js/form.js +3 -3
  108. slm/static/slm/js/formWidget.js +3 -3
  109. slm/static/slm/js/persistable.js +5 -1
  110. slm/templates/admin/base.html +1 -0
  111. slm/templates/rest_framework/base.html +23 -11
  112. slm/templates/slm/base.html +27 -22
  113. slm/templates/slm/forms/widgets/auto_complete.html +12 -11
  114. slm/templates/slm/forms/widgets/auto_complete_multiple.html +8 -7
  115. slm/templates/slm/station/download.html +6 -6
  116. slm/templates/slm/station/edit.html +9 -17
  117. slm/templates/slm/station/review.html +5 -3
  118. slm/templates/slm/station/upload.html +4 -1
  119. slm/templates/slm/station/uploads/legacy.html +1 -1
  120. slm/templates/slm/widgets/alert_scroll.html +4 -8
  121. slm/templates/slm/widgets/filelist.html +0 -5
  122. slm/templates/slm/widgets/log_scroll.html +2 -13
  123. slm/templates/slm/widgets/stationlist.html +2 -8
  124. slm/templatetags/slm.py +70 -9
  125. slm/utils.py +13 -4
  126. slm/validators.py +14 -14
  127. slm/views.py +6 -6
  128. slm/wsgi.py +16 -0
  129. igs_slm-0.1.4b0.dist-info/METADATA +0 -154
  130. igs_slm-0.1.4b0.dist-info/entry_points.txt +0 -3
  131. slm/bin/templates/{{ project_dir }}/sites/{{ site }}/validation.py +0 -11
  132. /slm/bin/templates/{{ project_dir }}/{sites → src/sites}/__init__.py +0 -0
  133. /slm/bin/templates/{{ project_dir }}/{sites → src/sites}/{{ site }}/__init__.py +0 -0
  134. /slm/bin/templates/{{ project_dir }}/{sites → src/sites}/{{ site }}/develop/local.py +0 -0
  135. /slm/bin/templates/{{ project_dir }}/{sites → src/sites}/{{ site }}/develop/wsgi.py +0 -0
  136. /slm/bin/templates/{{ project_dir }}/{sites → src/sites}/{{ site }}/manage.py +0 -0
  137. /slm/bin/templates/{{ project_dir }}/{sites → src/sites}/{{ site }}/production/wsgi.py +0 -0
  138. /slm/bin/templates/{{ project_dir }}/{sites → src/sites}/{{ site }}/urls.py +0 -0
  139. /slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/__init__.py +0 -0
  140. /slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/admin.py +0 -0
  141. /slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/apps.py +0 -0
  142. /slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/management/__init__.py +0 -0
  143. /slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/management/commands/__init__.py +0 -0
  144. /slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/management/commands/import_archive.py +0 -0
  145. /slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/migrations/__init__.py +0 -0
  146. /slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/models.py +0 -0
  147. /slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/templates/slm/base.html +0 -0
  148. /slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/urls.py +0 -0
  149. /slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/views.py +0 -0
slm/models/index.py CHANGED
@@ -1,8 +1,24 @@
1
1
  import os
2
+ import typing as t
3
+ from datetime import datetime
2
4
 
5
+ from django.contrib.postgres.constraints import ExclusionConstraint
6
+ from django.contrib.postgres.fields import DateTimeRangeField
7
+ from django.contrib.postgres.fields.ranges import DateTimeTZRange, RangeOperators
8
+ from django.core.exceptions import ValidationError
3
9
  from django.core.files.base import ContentFile
4
10
  from django.db import models, transaction
5
- from django.db.models import F, OuterRef, Q, Subquery, Value
11
+ from django.db.models import (
12
+ CharField,
13
+ DateTimeField,
14
+ Deferrable,
15
+ F,
16
+ Func,
17
+ OuterRef,
18
+ Q,
19
+ Subquery,
20
+ Value,
21
+ )
6
22
  from django.db.models.functions import (
7
23
  Cast,
8
24
  Concat,
@@ -14,6 +30,7 @@ from django.db.models.functions import (
14
30
  Now,
15
31
  Substr,
16
32
  )
33
+ from django.urls import reverse
17
34
  from django.utils.functional import cached_property
18
35
  from django.utils.timezone import now
19
36
  from django.utils.translation import gettext_lazy as _
@@ -33,6 +50,22 @@ from slm.parsing import xsd as xsd_parsing
33
50
 
34
51
 
35
52
  class ArchiveIndexManager(models.Manager):
53
+ def get_queryset(self):
54
+ """
55
+ All querysets will be ordered most recent entries first. We also annotated begin/end
56
+ datetimes onto the queryset. These annotations are also pre-indexed.
57
+ """
58
+ return (
59
+ ArchiveIndexQuerySet(self.model, using=self._db)
60
+ .annotate(
61
+ begin=Func(
62
+ "valid_range", function="lower", output_field=DateTimeField()
63
+ ),
64
+ end=Func("valid_range", function="upper", output_field=DateTimeField()),
65
+ )
66
+ .most_recent_first()
67
+ )
68
+
36
69
  def regenerate(self, site, log_format):
37
70
  """
38
71
  Regenerate the index file for this index. This will only be done for
@@ -42,7 +75,7 @@ class ArchiveIndexManager(models.Manager):
42
75
  :param log_format: The log format to regenerate.
43
76
  """
44
77
  new_file = ArchivedSiteLog.objects.from_index(
45
- index=self.get_queryset().filter(site=site).order_by("-begin").first(),
78
+ index=self.get_queryset().filter(site=site).first(),
46
79
  log_format=log_format,
47
80
  regenerate=True,
48
81
  )
@@ -50,7 +83,9 @@ class ArchiveIndexManager(models.Manager):
50
83
 
51
84
  def add_index(self, site, formats=list(SiteLogFormat)):
52
85
  assert site.last_publish, "last_publish must be set before calling add_index"
53
- existing = self.filter(site=site, begin=site.last_publish).first()
86
+ existing = self.filter(
87
+ site=site, valid_range__startswith=site.last_publish
88
+ ).first()
54
89
  if existing:
55
90
  return existing
56
91
 
@@ -69,14 +104,11 @@ class ArchiveIndexManager(models.Manager):
69
104
  if site.last_publish:
70
105
  last = (
71
106
  self.filter(site=site)
72
- .filter(
73
- Q(begin__lte=site.last_publish) & Q(end__isnull=True)
74
- | Q(end__gt=site.last_publish)
75
- )
107
+ .filter(valid_range__contains=site.last_publish)
76
108
  .first()
77
109
  )
78
110
  if last:
79
- last.end = site.last_publish
111
+ last.valid_range = DateTimeTZRange(last.begin, site.last_publish)
80
112
  last.save()
81
113
 
82
114
  def create(self, **kwargs):
@@ -90,34 +122,64 @@ class ArchiveIndexManager(models.Manager):
90
122
  Insert a new index into an existing index deck (i.e. between existing
91
123
  indexes).
92
124
  """
93
- existing = self.get_queryset().filter(site=site, begin=begin).first()
125
+ existing = (
126
+ self.get_queryset().filter(site=site, valid_range__startswith=begin).first()
127
+ )
94
128
  if existing:
95
129
  return existing
96
130
  next_index = (
97
131
  self.get_queryset()
98
- .filter(site=site, begin__gt=begin)
99
- .order_by("begin")
132
+ .filter(site=site, valid_range__startswith__gt=begin)
133
+ .oldest_first()
100
134
  .first()
101
135
  )
102
136
  prev_index = (
103
137
  self.get_queryset()
104
- .filter(site=site, begin__lt=begin)
105
- .order_by("-begin")
138
+ .filter(site=site, valid_range__startswith__lt=begin)
139
+ .most_recent_first()
106
140
  .first()
107
141
  )
108
- kwargs.setdefault("end", next_index.begin if next_index else None)
142
+
143
+ # Determine new index end (open-ended if no next)
144
+ kwargs.setdefault(
145
+ "valid_range",
146
+ DateTimeTZRange(begin, next_index.begin if next_index else None),
147
+ )
148
+
109
149
  if prev_index:
110
- prev_index.end = begin
150
+ prev_index.valid_range = DateTimeTZRange(prev_index.begin, begin)
111
151
  prev_index.save()
112
152
  return super().create(begin=begin, site=site, **kwargs)
113
153
 
114
154
 
115
155
  class ArchiveIndexQuerySet(models.QuerySet):
156
+ def most_recent_first(self, *fields: str):
157
+ """
158
+ Order rows decending in time and by any other given fields.
159
+
160
+ :param: fields - other columns to order by in order of priority
161
+ """
162
+ return self.order_by(Func("valid_range", function="lower").desc(), *fields)
163
+
164
+ def oldest_first(self, *fields: str):
165
+ """
166
+ Order rows ascending in time and by any other given fields.
167
+
168
+ :param: fields - other columns to order by in order of priority
169
+ """
170
+ return self.order_by(Func("valid_range", function="lower"), *fields)
171
+
116
172
  def delete(self):
117
173
  """
118
174
  We can't just delete an archive to remove it - we also have to
119
175
  update the end time of the previous archive in the index if there
120
176
  is one to reflect the end time of the deleted log
177
+
178
+ .. note::
179
+
180
+ The logic in this delete will only stich together immediately adjacent
181
+ index entries - that is entries that have no time gaps between their
182
+ ranges.
121
183
  """
122
184
  from slm.models import Site, SiteForm
123
185
 
@@ -143,17 +205,26 @@ class ArchiveIndexQuerySet(models.QuerySet):
143
205
  .filter(
144
206
  Q(site=OuterRef("site"))
145
207
  & Q(begin__gte=OuterRef("end"))
146
- & ~Q(pk__in=self)
208
+ & ~Q(pk__in=self.values("pk"))
147
209
  & Q(before__isnull=False)
148
210
  )
149
- .order_by("end")
211
+ .oldest_first()
150
212
  )
151
213
  adjacent_indexes = ArchiveIndex.objects.annotate(
152
214
  adjacent=Subquery(after.values("pk")[:1]),
153
- new_end=Subquery(new_bookends.values("begin")[:1]),
154
- ).filter(Q(adjacent__isnull=False) & ~Q(pk__in=self))
215
+ new_valid_range=Func(
216
+ Func(
217
+ F("valid_range"), function="lower", output_field=DateTimeField()
218
+ ),
219
+ Subquery(new_bookends.values("begin")[:1]),
220
+ function="tstzrange",
221
+ output_field=DateTimeRangeField(),
222
+ ),
223
+ ).filter(Q(adjacent__isnull=False) & ~Q(pk__in=self.values("pk")))
155
224
 
156
225
  # we use the new last indexes' old last end date
226
+ # TODO - to really do this right would have to set published state to unpublished
227
+ # where appropri
157
228
  forms = []
158
229
  for form in SiteForm.objects.filter(site__in=published_sites):
159
230
  form.pk = None
@@ -164,7 +235,7 @@ class ArchiveIndexQuerySet(models.QuerySet):
164
235
  SiteForm.objects.bulk_create(forms)
165
236
  published_sites.synchronize_denormalized_state()
166
237
 
167
- adjacent_indexes.update(end=F("new_end"))
238
+ adjacent_indexes.update(valid_range=F("new_valid_range"))
168
239
  deleted = super().delete()
169
240
  return deleted
170
241
 
@@ -172,7 +243,7 @@ class ArchiveIndexQuerySet(models.QuerySet):
172
243
  def epoch_q(epoch=None):
173
244
  if epoch is None:
174
245
  epoch = now()
175
- return Q(begin__lte=epoch) & (Q(end__gt=epoch) | Q(end__isnull=True))
246
+ return Q(valid_range__contains=epoch)
176
247
 
177
248
  def at_epoch(self, epoch=None):
178
249
  return self.filter(self.epoch_q(epoch))
@@ -199,7 +270,11 @@ class ArchiveIndexQuerySet(models.QuerySet):
199
270
  )
200
271
 
201
272
  def annotate_filenames(
202
- self, name_len=None, field_name="filename", lower_case=False, log_format=None
273
+ self,
274
+ name_len: t.Optional[int] = None,
275
+ field_name: str = "filename",
276
+ lower_case: bool = False,
277
+ log_format: bool = None,
203
278
  ):
204
279
  """
205
280
  Add the log names (w/o) extension as a property called filename to
@@ -221,48 +296,82 @@ class ArchiveIndexQuerySet(models.QuerySet):
221
296
  if lower_case:
222
297
  name_str = Lower(name_str)
223
298
 
299
+ begin = Func("valid_range", function="lower", output_field=DateTimeField())
300
+
224
301
  parts = [
225
302
  name_str,
226
303
  Value("_"),
227
- Cast(ExtractYear("begin"), models.CharField()),
228
- LPad(
229
- Cast(ExtractMonth("begin"), models.CharField()), 2, fill_text=Value("0")
230
- ),
304
+ Cast(ExtractYear(begin), models.CharField()),
231
305
  LPad(
232
- Cast(ExtractDay("begin"), models.CharField()), 2, fill_text=Value("0")
306
+ Cast(ExtractMonth(begin), models.CharField()), 2, fill_text=Value("0")
233
307
  ),
308
+ LPad(Cast(ExtractDay(begin), models.CharField()), 2, fill_text=Value("0")),
234
309
  ]
235
310
  if log_format:
236
311
  parts.append(Value(f".{log_format.ext}"))
237
312
 
238
- return self.annotate(**{field_name: Concat(*parts)})
313
+ return self.annotate(**{field_name: Concat(*parts, output_field=CharField())})
239
314
 
240
315
 
241
316
  class ArchiveIndex(models.Model):
317
+ """
318
+ The ArchiveIndex table stores references to serialized site log files indexed by the time
319
+ range in which they were current. The primary purpose of this table is to allow serialized
320
+ site log formats to change over time while maintaining a full historical record.
321
+ """
322
+
242
323
  site = models.ForeignKey(
243
324
  "slm.Site", on_delete=models.CASCADE, null=False, related_name="indexes"
244
325
  )
245
326
 
246
- # the point in time at which this record begins being valid
247
- begin = models.DateTimeField(
248
- null=False,
249
- db_index=True,
250
- help_text=_("The point in time at which this archive became valid."),
327
+ # bounds are inclusive/exclusive
328
+ valid_range = DateTimeRangeField(
329
+ null=True, blank=True, default_bounds="[)", db_index=True
251
330
  )
252
331
 
253
- # the point in time at which this record stops being valid
254
- end = models.DateTimeField(
255
- null=True,
256
- db_index=True,
257
- help_text=_("The point in time at which this archive stopped being valid."),
258
- )
332
+ @property
333
+ def begin(self) -> datetime:
334
+ return getattr(self, "_begin", self.valid_range.lower)
335
+
336
+ @begin.setter
337
+ def begin(self, begin: datetime):
338
+ self._begin = begin
339
+
340
+ @property
341
+ def end(self) -> t.Optional[datetime]:
342
+ return getattr(self, "_end", self.valid_range.upper)
343
+
344
+ @end.setter
345
+ def end(self, end: datetime):
346
+ self._end = end
259
347
 
260
348
  objects = ArchiveIndexManager.from_queryset(ArchiveIndexQuerySet)()
261
349
 
350
+ def clean(self):
351
+ super().clean()
352
+
353
+ # Skip validation if range is missing
354
+ if not self.valid_range or not self.site_id:
355
+ return
356
+
357
+ # Exclude self if updating
358
+ overlaps = ArchiveIndex.objects.filter(
359
+ site=self.site, valid_range__overlap=self.valid_range
360
+ )
361
+ if self.pk:
362
+ overlaps = overlaps.exclude(pk=self.pk)
363
+
364
+ if overlaps.exists():
365
+ overlapping_ranges = [f"[{o.begin}, {o.end or '∞'})" for o in overlaps]
366
+ message = _(
367
+ "This time range overlaps with an existing archive index for this site:\n"
368
+ ) + "\n".join(overlapping_ranges)
369
+ raise ValidationError({"valid_range": message})
370
+
262
371
  def __str__(self):
263
372
  return (
264
373
  f"{self.site.name} | {self.begin.date()} - "
265
- f'{self.end.date() if self.end else "present"}'
374
+ f"{self.end.date() if self.end else 'present'}"
266
375
  )
267
376
 
268
377
  def delete(self, using=None, keep_parents=False):
@@ -273,12 +382,20 @@ class ArchiveIndex(models.Model):
273
382
  return ArchiveIndex.objects.filter(pk=self.pk).delete()
274
383
 
275
384
  class Meta:
276
- ordering = ("-begin",)
385
+ ordering = ("-valid_range",)
277
386
  indexes = [
278
- models.Index(fields=("begin", "end")),
279
- models.Index(fields=("site", "begin", "end")),
387
+ models.Index(fields=("site", "valid_range")),
388
+ ]
389
+ constraints = [
390
+ ExclusionConstraint(
391
+ name="no_overlapping_ranges_per_site",
392
+ expressions=[
393
+ ("site", RangeOperators.EQUAL),
394
+ ("valid_range", RangeOperators.OVERLAPS),
395
+ ],
396
+ deferrable=Deferrable.DEFERRED,
397
+ )
280
398
  ]
281
- unique_together = (("site", "begin"),)
282
399
  verbose_name = "Archive Index"
283
400
  verbose_name_plural = "Archive Index"
284
401
 
@@ -300,6 +417,9 @@ class ArchivedSiteLogManager(models.Manager):
300
417
  return file
301
418
  elif file:
302
419
  file.delete()
420
+ filename = index.site.get_filename(
421
+ log_format=log_format, epoch=index.begin, lower_case=True
422
+ )
303
423
  return self.model.objects.create(
304
424
  site=index.site,
305
425
  log_format=log_format,
@@ -307,14 +427,12 @@ class ArchivedSiteLogManager(models.Manager):
307
427
  timestamp=index.begin,
308
428
  mimetype=log_format.mimetype,
309
429
  file_type=SLMFileType.SITE_LOG,
310
- name=index.site.get_filename(log_format=log_format, epoch=index.begin),
430
+ name=filename,
311
431
  file=ContentFile(
312
432
  SiteLogSerializer(instance=index.site)
313
433
  .format(log_format)
314
434
  .encode("utf-8"),
315
- name=index.site.get_filename(
316
- log_format=log_format, epoch=index.begin
317
- ),
435
+ name=filename,
318
436
  ),
319
437
  gml_version=(
320
438
  GeodesyMLVersion.latest()
@@ -333,7 +451,11 @@ class ArchivedSiteLogManager(models.Manager):
333
451
 
334
452
  class ArchivedSiteLogQuerySet(models.QuerySet):
335
453
  def annotate_filenames(
336
- self, name_len=None, field_name="filename", lower_case=False, log_format=None
454
+ self,
455
+ name_len: t.Optional[int] = None,
456
+ field_name: str = "filename",
457
+ lower_case: bool = False,
458
+ log_format: t.Optional[SiteLogFormat] = None,
337
459
  ):
338
460
  """
339
461
  Add the log names (w/o) extension as a property called filename to
@@ -355,17 +477,21 @@ class ArchivedSiteLogQuerySet(models.QuerySet):
355
477
  if lower_case:
356
478
  name_str = Lower(name_str)
357
479
 
480
+ index_begin = Func(
481
+ F("index__valid_range"), function="lower", output_field=DateTimeField()
482
+ )
483
+
358
484
  parts = [
359
485
  name_str,
360
486
  Value("_"),
361
- Cast(ExtractYear("index__begin"), models.CharField()),
487
+ Cast(ExtractYear(index_begin), models.CharField()),
362
488
  LPad(
363
- Cast(ExtractMonth("index__begin"), models.CharField()),
489
+ Cast(ExtractMonth(index_begin), models.CharField()),
364
490
  2,
365
491
  fill_text=Value("0"),
366
492
  ),
367
493
  LPad(
368
- Cast(ExtractDay("index__begin"), models.CharField()),
494
+ Cast(ExtractDay(index_begin), models.CharField()),
369
495
  2,
370
496
  fill_text=Value("0"),
371
497
  ),
@@ -382,7 +508,7 @@ class ArchivedSiteLog(SiteFile):
382
508
  ArchiveIndex, on_delete=models.CASCADE, related_name="files"
383
509
  )
384
510
 
385
- name = models.CharField(max_length=50)
511
+ name = models.CharField(max_length=50, db_index=True)
386
512
 
387
513
  objects = ArchivedSiteLogManager.from_queryset(ArchivedSiteLogQuerySet)()
388
514
 
@@ -391,6 +517,10 @@ class ArchivedSiteLog(SiteFile):
391
517
  return f"[{self.index}] {self.name}"
392
518
  return f"[{self.index}] {os.path.basename(self.file.path)}"
393
519
 
520
+ @cached_property
521
+ def link(self):
522
+ return reverse("slm_public_api:archive-detail", kwargs={"pk": self.pk})
523
+
394
524
  def parse(self) -> BaseBinder:
395
525
  if self.log_format is SiteLogFormat.GEODESY_ML:
396
526
  return xsd_parsing.SiteLogBinder(
slm/models/sitelog.py CHANGED
@@ -9,7 +9,7 @@ from django.contrib.auth import get_user_model
9
9
  from django.contrib.auth.models import Permission
10
10
  from django.contrib.gis.db import models as gis_models
11
11
  from django.core.exceptions import FieldDoesNotExist, ValidationError
12
- from django.core.validators import MaxValueValidator, MinValueValidator, RegexValidator
12
+ from django.core.validators import MaxValueValidator, MinValueValidator
13
13
  from django.db import models, transaction
14
14
  from django.db.models import (
15
15
  CheckConstraint,
@@ -53,6 +53,7 @@ from slm.defines import (
53
53
  SiteLogStatus,
54
54
  TectonicPlates,
55
55
  )
56
+ from slm.models.fields import StationNameField
56
57
  from slm.utils import date_to_str
57
58
  from slm.validators import get_validators
58
59
 
@@ -83,9 +84,7 @@ class SubquerySum(Subquery):
83
84
  output_field = models.IntegerField()
84
85
 
85
86
  def __init__(self, *args, **kwargs):
86
- self.template = (
87
- f'(SELECT SUM({kwargs.pop("field")}) ' f"FROM (%(subquery)s) _sum)"
88
- )
87
+ self.template = f"(SELECT SUM({kwargs.pop('field')}) FROM (%(subquery)s) _sum)"
89
88
  super().__init__(*args, **kwargs)
90
89
 
91
90
 
@@ -118,7 +117,7 @@ class SiteQuerySet(models.QuerySet):
118
117
 
119
118
  latest_archive = ArchivedSiteLog.objects.filter(
120
119
  Q(site=OuterRef("pk")) & Q(log_format=log_format)
121
- ).order_by("-index__begin")
120
+ ).order_by("-index__valid_range")
122
121
  size_field = f"{log_format.ext if prefix is None else prefix}_size"
123
122
  file_field = f"{log_format.ext if prefix is None else prefix}_file"
124
123
  return self.annotate(
@@ -651,17 +650,11 @@ class Site(models.Model):
651
650
 
652
651
  objects = SiteManager.from_queryset(SiteQuerySet)()
653
652
 
654
- name = models.CharField(
655
- max_length=9,
653
+ name = StationNameField(
654
+ max_length=50,
656
655
  unique=True,
657
- help_text=_(
658
- "This is the 9 Character station name (XXXXMRCCC) used in RINEX 3 "
659
- "filenames Format: (XXXX - existing four character IGS station "
660
- "name, M - Monument or marker number (0-9), R - Receiver number "
661
- "(0-9), CCC - Three digit ISO 3166-1 country code)"
662
- ),
656
+ help_text=_("The name of the station."),
663
657
  db_index=True,
664
- validators=[RegexValidator(r"[\w]{4}[\d]{2}[\w]{3}")],
665
658
  )
666
659
 
667
660
  # todo can site exist without agency?
@@ -779,6 +772,7 @@ class Site(models.Model):
779
772
  (default False)
780
773
  :return: The filename including extension.
781
774
  """
775
+ # TODO - make this pluggable
782
776
  if epoch is None:
783
777
  epoch = self.last_publish or self.last_update or self.created
784
778
  if name_len is None and log_format is SiteLogFormat.LEGACY:
@@ -890,15 +884,19 @@ class Site(models.Model):
890
884
  return user.agencies.filter(pk__in=self.agencies.all()).count() > 0
891
885
  return False
892
886
 
893
- def update_status(self, save=True, user=None, timestamp=None, first_publish=False):
887
+ def update_status(
888
+ self, save=True, user=None, timestamp=None, first_publish=False, reverted=False
889
+ ):
894
890
  """
895
891
  Update the denormalized data that is too expensive to query on the
896
892
  fly. This includes flag count, moderation status and DateTimes. Also
897
893
  check for and delete any review requests if a publish was done.
898
894
 
899
- :param save:
895
+ :param save: If true the site row will be saved to the database.
900
896
  :param user: The user responsible for a status update check
901
897
  :param timestamp: The time at which the status update is triggered
898
+ :param first_publish: True if this is the first time the site log is being published.
899
+ :param reverted: A boolean indicating if this update was triggered by a reversion or not.
902
900
  :return:
903
901
  """
904
902
  if not timestamp:
@@ -926,6 +924,9 @@ class Site(models.Model):
926
924
  self.review_request.delete()
927
925
 
928
926
  if save:
927
+ setattr( # ugly but we need to pass info to the post_save signal handler somehow
928
+ self, "_reverted", reverted
929
+ )
929
930
  self.save()
930
931
 
931
932
  def revert(self):
@@ -933,7 +934,7 @@ class Site(models.Model):
933
934
  for section in self.sections():
934
935
  reverted |= getattr(self, section.accessor).revert()
935
936
  if reverted:
936
- self.update_status()
937
+ self.update_status(reverted=reverted)
937
938
  return reverted
938
939
 
939
940
  def published(self, epoch=None):
@@ -1194,6 +1195,9 @@ class SiteSectionQueryset(gis_models.QuerySet):
1194
1195
  def sort(self, reverse=False):
1195
1196
  return self
1196
1197
 
1198
+ class Meta:
1199
+ order_by = ("name",)
1200
+
1197
1201
 
1198
1202
  class SiteLocationManager(SiteSectionManager):
1199
1203
  pass
@@ -1260,7 +1264,7 @@ class SiteSection(gis_models.Model):
1260
1264
  ).delete()[0]
1261
1265
  )
1262
1266
  if reverted:
1263
- self.site.update_status()
1267
+ self.site.update_status(reverted=reverted)
1264
1268
  return reverted
1265
1269
 
1266
1270
  @property
@@ -1694,7 +1698,7 @@ class SiteSubSection(SiteSection):
1694
1698
  & Q(subsection=self.subsection)
1695
1699
  ).update(is_deleted=False)
1696
1700
  if reverted:
1697
- self.site.update_status()
1701
+ self.site.update_status(reverted=reverted)
1698
1702
  return reverted
1699
1703
 
1700
1704
  @cached_property
@@ -2754,7 +2758,7 @@ class SiteAntenna(SiteSubSection):
2754
2758
  blank=False,
2755
2759
  verbose_name=_("Date Installed (UTC)"),
2756
2760
  help_text=_(
2757
- "Enter the date the receiver was installed. " "Format: (CCYY-MM-DDThh:mmZ)"
2761
+ "Enter the date the receiver was installed. Format: (CCYY-MM-DDThh:mmZ)"
2758
2762
  ),
2759
2763
  db_index=True,
2760
2764
  )
@@ -2999,8 +3003,7 @@ class SiteFrequencyStandard(SiteSubSection):
2999
3003
  def effective(self):
3000
3004
  if self.effective_start and self.effective_end:
3001
3005
  return (
3002
- f"{date_to_str(self.effective_start)}/"
3003
- f"{date_to_str(self.effective_end)}"
3006
+ f"{date_to_str(self.effective_start)}/{date_to_str(self.effective_end)}"
3004
3007
  )
3005
3008
  elif self.effective_start:
3006
3009
  return f"{date_to_str(self.effective_start)}"
@@ -3115,8 +3118,7 @@ class SiteCollocation(SiteSubSection):
3115
3118
  def effective(self):
3116
3119
  if self.effective_start and self.effective_end:
3117
3120
  return (
3118
- f"{date_to_str(self.effective_start)}/"
3119
- f"{date_to_str(self.effective_end)}"
3121
+ f"{date_to_str(self.effective_start)}/{date_to_str(self.effective_end)}"
3120
3122
  )
3121
3123
  elif self.effective_start:
3122
3124
  return f"{date_to_str(self.effective_start)}"
@@ -3229,8 +3231,7 @@ class MeteorologicalInstrumentation(SiteSubSection):
3229
3231
  def effective(self):
3230
3232
  if self.effective_start and self.effective_end:
3231
3233
  return (
3232
- f"{date_to_str(self.effective_start)}/"
3233
- f"{date_to_str(self.effective_end)}"
3234
+ f"{date_to_str(self.effective_start)}/{date_to_str(self.effective_end)}"
3234
3235
  )
3235
3236
  elif self.effective_start:
3236
3237
  return f"{date_to_str(self.effective_start)}"
@@ -3293,7 +3294,7 @@ class MeteorologicalInstrumentation(SiteSubSection):
3293
3294
  blank=False,
3294
3295
  null=True,
3295
3296
  help_text=_(
3296
- "Enter the effective start date for the sensor. " "Format: (CCYY-MM-DD)"
3297
+ "Enter the effective start date for the sensor. Format: (CCYY-MM-DD)"
3297
3298
  ),
3298
3299
  db_index=True,
3299
3300
  )
@@ -3302,7 +3303,7 @@ class MeteorologicalInstrumentation(SiteSubSection):
3302
3303
  blank=True,
3303
3304
  default=None,
3304
3305
  help_text=_(
3305
- "Enter the effective end date for the sensor. " "Format: (CCYY-MM-DD)"
3306
+ "Enter the effective end date for the sensor. Format: (CCYY-MM-DD)"
3306
3307
  ),
3307
3308
  db_index=True,
3308
3309
  )
@@ -3404,8 +3405,7 @@ class SiteHumiditySensor(MeteorologicalInstrumentation):
3404
3405
  max_length=50,
3405
3406
  verbose_name=_("Aspiration"),
3406
3407
  help_text=_(
3407
- "Enter the aspiration type if known. "
3408
- "Format: (UNASPIRATED/NATURAL/FAN/etc)"
3408
+ "Enter the aspiration type if known. Format: (UNASPIRATED/NATURAL/FAN/etc)"
3409
3409
  ),
3410
3410
  db_index=True,
3411
3411
  )
@@ -3546,8 +3546,7 @@ class SiteTemperatureSensor(MeteorologicalInstrumentation):
3546
3546
  max_length=50,
3547
3547
  verbose_name=_("Aspiration"),
3548
3548
  help_text=_(
3549
- "Enter the aspiration type if known. "
3550
- "Format: (UNASPIRATED/NATURAL/FAN/etc)"
3549
+ "Enter the aspiration type if known. Format: (UNASPIRATED/NATURAL/FAN/etc)"
3551
3550
  ),
3552
3551
  db_index=True,
3553
3552
  )
@@ -3664,8 +3663,7 @@ class Condition(SiteSubSection):
3664
3663
  def effective(self):
3665
3664
  if self.effective_start and self.effective_end:
3666
3665
  return (
3667
- f"{date_to_str(self.effective_start)}/"
3668
- f"{date_to_str(self.effective_end)}"
3666
+ f"{date_to_str(self.effective_start)}/{date_to_str(self.effective_end)}"
3669
3667
  )
3670
3668
  elif self.effective_start:
3671
3669
  return f"{date_to_str(self.effective_start)}"
@@ -3684,7 +3682,7 @@ class Condition(SiteSubSection):
3684
3682
  null=True,
3685
3683
  default=None,
3686
3684
  help_text=_(
3687
- "Enter the effective start date for the condition. " "Format: (CCYY-MM-DD)"
3685
+ "Enter the effective start date for the condition. Format: (CCYY-MM-DD)"
3688
3686
  ),
3689
3687
  db_index=True,
3690
3688
  )
@@ -3694,7 +3692,7 @@ class Condition(SiteSubSection):
3694
3692
  null=True,
3695
3693
  default=None,
3696
3694
  help_text=_(
3697
- "Enter the effective end date for the condition. " "Format: (CCYY-MM-DD)"
3695
+ "Enter the effective end date for the condition. Format: (CCYY-MM-DD)"
3698
3696
  ),
3699
3697
  db_index=True,
3700
3698
  )
@@ -3859,8 +3857,7 @@ class SiteLocalEpisodicEffects(SiteSubSection):
3859
3857
  def effective(self):
3860
3858
  if self.effective_start and self.effective_end:
3861
3859
  return (
3862
- f"{date_to_str(self.effective_start)}/"
3863
- f"{date_to_str(self.effective_end)}"
3860
+ f"{date_to_str(self.effective_start)}/{date_to_str(self.effective_end)}"
3864
3861
  )
3865
3862
  elif self.effective_start:
3866
3863
  return f"{date_to_str(self.effective_start)}"