igs-slm 0.1.4b0__py3-none-any.whl → 0.1.5b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- igs_slm-0.1.5b1.dist-info/METADATA +115 -0
- {igs_slm-0.1.4b0.dist-info → igs_slm-0.1.5b1.dist-info}/RECORD +193 -173
- {igs_slm-0.1.4b0.dist-info → igs_slm-0.1.5b1.dist-info}/WHEEL +1 -1
- igs_slm-0.1.5b1.dist-info/entry_points.txt +3 -0
- {igs_slm-0.1.4b0.dist-info → igs_slm-0.1.5b1.dist-info/licenses}/LICENSE +1 -1
- slm/__init__.py +17 -14
- slm/admin.py +32 -5
- slm/api/edit/views.py +22 -9
- slm/api/public/views.py +9 -7
- slm/api/views.py +45 -6
- slm/apps.py +28 -6
- slm/authentication.py +3 -2
- slm/bin/startproject.py +102 -31
- slm/bin/templates/{{ project_dir }}/pyproject.toml +30 -21
- slm/bin/templates/{{ project_dir }}/{sites → src/sites}/{{ site }}/base.py +12 -1
- slm/bin/templates/{{ project_dir }}/{sites → src/sites}/{{ site }}/develop/__init__.py +5 -27
- slm/bin/templates/{{ project_dir }}/{sites → src/sites}/{{ site }}/production/__init__.py +6 -27
- slm/bin/templates/{{ project_dir }}/src/sites/{{ site }}/urls.py +15 -0
- slm/bin/templates/{{ project_dir }}/src/sites/{{ site }}/validation.py +29 -0
- slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/urls.py +1 -0
- slm/context.py +5 -0
- slm/defines/AlertLevel.py +10 -2
- slm/defines/AntennaCalibration.py +6 -4
- slm/defines/AntennaFeatures.py +12 -9
- slm/defines/AntennaReferencePoint.py +12 -10
- slm/defines/Aspiration.py +4 -2
- slm/defines/CardinalDirection.py +6 -4
- slm/defines/CollocationStatus.py +3 -1
- slm/defines/EquipmentState.py +6 -12
- slm/defines/FlagSeverity.py +4 -2
- slm/defines/FractureSpacing.py +7 -5
- slm/defines/FrequencyStandardType.py +6 -4
- slm/defines/GeodesyMLVersion.py +2 -0
- slm/defines/Instrumentation.py +9 -7
- slm/defines/LogEntryType.py +17 -15
- slm/defines/SLMFileType.py +4 -2
- slm/defines/SiteFileUploadStatus.py +7 -24
- slm/defines/SiteLogFormat.py +8 -32
- slm/defines/SiteLogStatus.py +9 -28
- slm/defines/TectonicPlates.py +18 -16
- slm/manage.py +24 -0
- slm/management/commands/check_upgrade.py +142 -0
- slm/management/commands/generate_sinex.py +110 -92
- slm/management/commands/head_from_index.py +11 -8
- slm/management/commands/import_archive.py +27 -18
- slm/management/commands/import_equipment.py +1 -1
- slm/management/commands/sitelog.py +1 -3
- slm/management/commands/synchronize.py +1 -1
- slm/management/commands/validate_db.py +6 -4
- slm/map/defines.py +18 -14
- slm/map/templates/slm/map.html +5 -7
- slm/migrations/0001_remove_archiveindex_no_overlapping_ranges_per_site_and_more.py +26 -0
- slm/migrations/0002_alter_archivedsitelog_file_and_more.py +44 -0
- slm/migrations/0003_alter_archivedsitelog_name_and_more.py +35 -0
- slm/migrations/0004_alter_site_name.py +24 -0
- slm/migrations/0005_slmversion.py +30 -0
- slm/migrations/0017_alter_logentry_unique_together_and_more.py +3 -1
- slm/migrations/0018_afix_deleted.py +3 -1
- slm/migrations/0018_alter_siteantenna_options_and_more.py +87 -56
- slm/migrations/0019_remove_siteantenna_marker_enu_siteantenna_marker_une_and_more.py +1 -1
- slm/migrations/0023_archivedsitelog_gml_version_and_more.py +1 -1
- slm/migrations/0031_alter_antenna_features.py +44 -0
- slm/migrations/0032_archiveindex_valid_range_and_more.py +84 -0
- slm/migrations/add_index_order_index.py +54 -0
- slm/migrations/normalize_index.py +147 -0
- slm/migrations/simplify_index.py +48 -0
- slm/migrations/verify_index.py +67 -0
- slm/models/__init__.py +2 -0
- slm/models/alerts.py +7 -10
- slm/models/data.py +1 -2
- slm/models/equipment.py +1 -1
- slm/models/fields.py +41 -0
- slm/models/index.py +183 -53
- slm/models/sitelog.py +35 -38
- slm/models/system.py +72 -31
- slm/models/user.py +1 -1
- slm/parsing/__init__.py +33 -16
- slm/parsing/legacy/binding.py +65 -34
- slm/parsing/legacy/parser.py +2 -2
- slm/parsing/xsd/binding.py +1 -1
- slm/parsing/xsd/parser.py +1 -2
- slm/receivers/__init__.py +2 -2
- slm/receivers/index.py +2 -1
- slm/receivers/migration.py +21 -0
- slm/settings/__init__.py +192 -4
- slm/settings/assets.py +26 -0
- slm/settings/auth.py +18 -14
- slm/settings/ckeditor.py +12 -6
- slm/settings/debug.py +2 -2
- slm/settings/emails.py +50 -0
- slm/settings/internationalization.py +8 -6
- slm/settings/logging.py +100 -88
- slm/settings/platform/darwin.py +16 -6
- slm/settings/rest.py +20 -15
- slm/settings/root.py +192 -98
- slm/settings/routines.py +5 -1
- slm/settings/secrets.py +20 -31
- slm/settings/security.py +7 -5
- slm/settings/slm.py +35 -23
- slm/settings/static_templates.py +12 -9
- slm/settings/templates.py +31 -25
- slm/settings/uploads.py +33 -5
- slm/settings/urls.py +7 -12
- slm/settings/validation.py +165 -165
- slm/signals.py +3 -2
- slm/static/slm/css/style.css +37 -36
- slm/static/slm/js/autocomplete.js +6 -4
- slm/static/slm/js/enums.js +6 -5
- slm/static/slm/js/file_modal.js +62 -0
- slm/static/slm/js/form.js +3 -3
- slm/static/slm/js/formWidget.js +3 -3
- slm/static/slm/js/persistable.js +5 -1
- slm/templates/admin/base.html +1 -0
- slm/templates/rest_framework/base.html +23 -11
- slm/templates/slm/base.html +27 -22
- slm/templates/slm/forms/widgets/auto_complete.html +12 -11
- slm/templates/slm/forms/widgets/auto_complete_multiple.html +8 -7
- slm/templates/slm/station/download.html +6 -6
- slm/templates/slm/station/edit.html +9 -17
- slm/templates/slm/station/review.html +5 -3
- slm/templates/slm/station/upload.html +4 -1
- slm/templates/slm/station/uploads/legacy.html +1 -1
- slm/templates/slm/widgets/alert_scroll.html +4 -8
- slm/templates/slm/widgets/filelist.html +0 -5
- slm/templates/slm/widgets/log_scroll.html +2 -13
- slm/templates/slm/widgets/stationlist.html +2 -8
- slm/templatetags/slm.py +70 -9
- slm/utils.py +13 -4
- slm/validators.py +14 -14
- slm/views.py +6 -6
- slm/wsgi.py +16 -0
- igs_slm-0.1.4b0.dist-info/METADATA +0 -154
- igs_slm-0.1.4b0.dist-info/entry_points.txt +0 -3
- slm/bin/templates/{{ project_dir }}/sites/{{ site }}/urls.py +0 -7
- slm/bin/templates/{{ project_dir }}/sites/{{ site }}/validation.py +0 -11
- /slm/bin/templates/{{ project_dir }}/{sites → src/sites}/__init__.py +0 -0
- /slm/bin/templates/{{ project_dir }}/{sites → src/sites}/{{ site }}/__init__.py +0 -0
- /slm/bin/templates/{{ project_dir }}/{sites → src/sites}/{{ site }}/develop/local.py +0 -0
- /slm/bin/templates/{{ project_dir }}/{sites → src/sites}/{{ site }}/develop/wsgi.py +0 -0
- /slm/bin/templates/{{ project_dir }}/{sites → src/sites}/{{ site }}/manage.py +0 -0
- /slm/bin/templates/{{ project_dir }}/{sites → src/sites}/{{ site }}/production/wsgi.py +0 -0
- /slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/__init__.py +0 -0
- /slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/admin.py +0 -0
- /slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/apps.py +0 -0
- /slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/management/__init__.py +0 -0
- /slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/management/commands/__init__.py +0 -0
- /slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/management/commands/import_archive.py +0 -0
- /slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/migrations/__init__.py +0 -0
- /slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/models.py +0 -0
- /slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/templates/slm/base.html +0 -0
- /slm/bin/templates/{{ project_dir }}/{{{ extension_app }} → src/{{ extension_app }}}/views.py +0 -0
slm/models/index.py
CHANGED
|
@@ -1,8 +1,24 @@
|
|
|
1
1
|
import os
|
|
2
|
+
import typing as t
|
|
3
|
+
from datetime import datetime
|
|
2
4
|
|
|
5
|
+
from django.contrib.postgres.constraints import ExclusionConstraint
|
|
6
|
+
from django.contrib.postgres.fields import DateTimeRangeField
|
|
7
|
+
from django.contrib.postgres.fields.ranges import DateTimeTZRange, RangeOperators
|
|
8
|
+
from django.core.exceptions import ValidationError
|
|
3
9
|
from django.core.files.base import ContentFile
|
|
4
10
|
from django.db import models, transaction
|
|
5
|
-
from django.db.models import
|
|
11
|
+
from django.db.models import (
|
|
12
|
+
CharField,
|
|
13
|
+
DateTimeField,
|
|
14
|
+
Deferrable,
|
|
15
|
+
F,
|
|
16
|
+
Func,
|
|
17
|
+
OuterRef,
|
|
18
|
+
Q,
|
|
19
|
+
Subquery,
|
|
20
|
+
Value,
|
|
21
|
+
)
|
|
6
22
|
from django.db.models.functions import (
|
|
7
23
|
Cast,
|
|
8
24
|
Concat,
|
|
@@ -14,6 +30,7 @@ from django.db.models.functions import (
|
|
|
14
30
|
Now,
|
|
15
31
|
Substr,
|
|
16
32
|
)
|
|
33
|
+
from django.urls import reverse
|
|
17
34
|
from django.utils.functional import cached_property
|
|
18
35
|
from django.utils.timezone import now
|
|
19
36
|
from django.utils.translation import gettext_lazy as _
|
|
@@ -33,6 +50,22 @@ from slm.parsing import xsd as xsd_parsing
|
|
|
33
50
|
|
|
34
51
|
|
|
35
52
|
class ArchiveIndexManager(models.Manager):
|
|
53
|
+
def get_queryset(self):
|
|
54
|
+
"""
|
|
55
|
+
All querysets will be ordered most recent entries first. We also annotated begin/end
|
|
56
|
+
datetimes onto the queryset. These annotations are also pre-indexed.
|
|
57
|
+
"""
|
|
58
|
+
return (
|
|
59
|
+
ArchiveIndexQuerySet(self.model, using=self._db)
|
|
60
|
+
.annotate(
|
|
61
|
+
begin=Func(
|
|
62
|
+
"valid_range", function="lower", output_field=DateTimeField()
|
|
63
|
+
),
|
|
64
|
+
end=Func("valid_range", function="upper", output_field=DateTimeField()),
|
|
65
|
+
)
|
|
66
|
+
.most_recent_first()
|
|
67
|
+
)
|
|
68
|
+
|
|
36
69
|
def regenerate(self, site, log_format):
|
|
37
70
|
"""
|
|
38
71
|
Regenerate the index file for this index. This will only be done for
|
|
@@ -42,7 +75,7 @@ class ArchiveIndexManager(models.Manager):
|
|
|
42
75
|
:param log_format: The log format to regenerate.
|
|
43
76
|
"""
|
|
44
77
|
new_file = ArchivedSiteLog.objects.from_index(
|
|
45
|
-
index=self.get_queryset().filter(site=site).
|
|
78
|
+
index=self.get_queryset().filter(site=site).first(),
|
|
46
79
|
log_format=log_format,
|
|
47
80
|
regenerate=True,
|
|
48
81
|
)
|
|
@@ -50,7 +83,9 @@ class ArchiveIndexManager(models.Manager):
|
|
|
50
83
|
|
|
51
84
|
def add_index(self, site, formats=list(SiteLogFormat)):
|
|
52
85
|
assert site.last_publish, "last_publish must be set before calling add_index"
|
|
53
|
-
existing = self.filter(
|
|
86
|
+
existing = self.filter(
|
|
87
|
+
site=site, valid_range__startswith=site.last_publish
|
|
88
|
+
).first()
|
|
54
89
|
if existing:
|
|
55
90
|
return existing
|
|
56
91
|
|
|
@@ -69,14 +104,11 @@ class ArchiveIndexManager(models.Manager):
|
|
|
69
104
|
if site.last_publish:
|
|
70
105
|
last = (
|
|
71
106
|
self.filter(site=site)
|
|
72
|
-
.filter(
|
|
73
|
-
Q(begin__lte=site.last_publish) & Q(end__isnull=True)
|
|
74
|
-
| Q(end__gt=site.last_publish)
|
|
75
|
-
)
|
|
107
|
+
.filter(valid_range__contains=site.last_publish)
|
|
76
108
|
.first()
|
|
77
109
|
)
|
|
78
110
|
if last:
|
|
79
|
-
last.
|
|
111
|
+
last.valid_range = DateTimeTZRange(last.begin, site.last_publish)
|
|
80
112
|
last.save()
|
|
81
113
|
|
|
82
114
|
def create(self, **kwargs):
|
|
@@ -90,34 +122,64 @@ class ArchiveIndexManager(models.Manager):
|
|
|
90
122
|
Insert a new index into an existing index deck (i.e. between existing
|
|
91
123
|
indexes).
|
|
92
124
|
"""
|
|
93
|
-
existing =
|
|
125
|
+
existing = (
|
|
126
|
+
self.get_queryset().filter(site=site, valid_range__startswith=begin).first()
|
|
127
|
+
)
|
|
94
128
|
if existing:
|
|
95
129
|
return existing
|
|
96
130
|
next_index = (
|
|
97
131
|
self.get_queryset()
|
|
98
|
-
.filter(site=site,
|
|
99
|
-
.
|
|
132
|
+
.filter(site=site, valid_range__startswith__gt=begin)
|
|
133
|
+
.oldest_first()
|
|
100
134
|
.first()
|
|
101
135
|
)
|
|
102
136
|
prev_index = (
|
|
103
137
|
self.get_queryset()
|
|
104
|
-
.filter(site=site,
|
|
105
|
-
.
|
|
138
|
+
.filter(site=site, valid_range__startswith__lt=begin)
|
|
139
|
+
.most_recent_first()
|
|
106
140
|
.first()
|
|
107
141
|
)
|
|
108
|
-
|
|
142
|
+
|
|
143
|
+
# Determine new index end (open-ended if no next)
|
|
144
|
+
kwargs.setdefault(
|
|
145
|
+
"valid_range",
|
|
146
|
+
DateTimeTZRange(begin, next_index.begin if next_index else None),
|
|
147
|
+
)
|
|
148
|
+
|
|
109
149
|
if prev_index:
|
|
110
|
-
prev_index.
|
|
150
|
+
prev_index.valid_range = DateTimeTZRange(prev_index.begin, begin)
|
|
111
151
|
prev_index.save()
|
|
112
152
|
return super().create(begin=begin, site=site, **kwargs)
|
|
113
153
|
|
|
114
154
|
|
|
115
155
|
class ArchiveIndexQuerySet(models.QuerySet):
|
|
156
|
+
def most_recent_first(self, *fields: str):
|
|
157
|
+
"""
|
|
158
|
+
Order rows decending in time and by any other given fields.
|
|
159
|
+
|
|
160
|
+
:param: fields - other columns to order by in order of priority
|
|
161
|
+
"""
|
|
162
|
+
return self.order_by(Func("valid_range", function="lower").desc(), *fields)
|
|
163
|
+
|
|
164
|
+
def oldest_first(self, *fields: str):
|
|
165
|
+
"""
|
|
166
|
+
Order rows ascending in time and by any other given fields.
|
|
167
|
+
|
|
168
|
+
:param: fields - other columns to order by in order of priority
|
|
169
|
+
"""
|
|
170
|
+
return self.order_by(Func("valid_range", function="lower"), *fields)
|
|
171
|
+
|
|
116
172
|
def delete(self):
|
|
117
173
|
"""
|
|
118
174
|
We can't just delete an archive to remove it - we also have to
|
|
119
175
|
update the end time of the previous archive in the index if there
|
|
120
176
|
is one to reflect the end time of the deleted log
|
|
177
|
+
|
|
178
|
+
.. note::
|
|
179
|
+
|
|
180
|
+
The logic in this delete will only stich together immediately adjacent
|
|
181
|
+
index entries - that is entries that have no time gaps between their
|
|
182
|
+
ranges.
|
|
121
183
|
"""
|
|
122
184
|
from slm.models import Site, SiteForm
|
|
123
185
|
|
|
@@ -143,17 +205,26 @@ class ArchiveIndexQuerySet(models.QuerySet):
|
|
|
143
205
|
.filter(
|
|
144
206
|
Q(site=OuterRef("site"))
|
|
145
207
|
& Q(begin__gte=OuterRef("end"))
|
|
146
|
-
& ~Q(pk__in=self)
|
|
208
|
+
& ~Q(pk__in=self.values("pk"))
|
|
147
209
|
& Q(before__isnull=False)
|
|
148
210
|
)
|
|
149
|
-
.
|
|
211
|
+
.oldest_first()
|
|
150
212
|
)
|
|
151
213
|
adjacent_indexes = ArchiveIndex.objects.annotate(
|
|
152
214
|
adjacent=Subquery(after.values("pk")[:1]),
|
|
153
|
-
|
|
154
|
-
|
|
215
|
+
new_valid_range=Func(
|
|
216
|
+
Func(
|
|
217
|
+
F("valid_range"), function="lower", output_field=DateTimeField()
|
|
218
|
+
),
|
|
219
|
+
Subquery(new_bookends.values("begin")[:1]),
|
|
220
|
+
function="tstzrange",
|
|
221
|
+
output_field=DateTimeRangeField(),
|
|
222
|
+
),
|
|
223
|
+
).filter(Q(adjacent__isnull=False) & ~Q(pk__in=self.values("pk")))
|
|
155
224
|
|
|
156
225
|
# we use the new last indexes' old last end date
|
|
226
|
+
# TODO - to really do this right would have to set published state to unpublished
|
|
227
|
+
# where appropri
|
|
157
228
|
forms = []
|
|
158
229
|
for form in SiteForm.objects.filter(site__in=published_sites):
|
|
159
230
|
form.pk = None
|
|
@@ -164,7 +235,7 @@ class ArchiveIndexQuerySet(models.QuerySet):
|
|
|
164
235
|
SiteForm.objects.bulk_create(forms)
|
|
165
236
|
published_sites.synchronize_denormalized_state()
|
|
166
237
|
|
|
167
|
-
adjacent_indexes.update(
|
|
238
|
+
adjacent_indexes.update(valid_range=F("new_valid_range"))
|
|
168
239
|
deleted = super().delete()
|
|
169
240
|
return deleted
|
|
170
241
|
|
|
@@ -172,7 +243,7 @@ class ArchiveIndexQuerySet(models.QuerySet):
|
|
|
172
243
|
def epoch_q(epoch=None):
|
|
173
244
|
if epoch is None:
|
|
174
245
|
epoch = now()
|
|
175
|
-
return Q(
|
|
246
|
+
return Q(valid_range__contains=epoch)
|
|
176
247
|
|
|
177
248
|
def at_epoch(self, epoch=None):
|
|
178
249
|
return self.filter(self.epoch_q(epoch))
|
|
@@ -199,7 +270,11 @@ class ArchiveIndexQuerySet(models.QuerySet):
|
|
|
199
270
|
)
|
|
200
271
|
|
|
201
272
|
def annotate_filenames(
|
|
202
|
-
self,
|
|
273
|
+
self,
|
|
274
|
+
name_len: t.Optional[int] = None,
|
|
275
|
+
field_name: str = "filename",
|
|
276
|
+
lower_case: bool = False,
|
|
277
|
+
log_format: bool = None,
|
|
203
278
|
):
|
|
204
279
|
"""
|
|
205
280
|
Add the log names (w/o) extension as a property called filename to
|
|
@@ -221,48 +296,82 @@ class ArchiveIndexQuerySet(models.QuerySet):
|
|
|
221
296
|
if lower_case:
|
|
222
297
|
name_str = Lower(name_str)
|
|
223
298
|
|
|
299
|
+
begin = Func("valid_range", function="lower", output_field=DateTimeField())
|
|
300
|
+
|
|
224
301
|
parts = [
|
|
225
302
|
name_str,
|
|
226
303
|
Value("_"),
|
|
227
|
-
Cast(ExtractYear(
|
|
228
|
-
LPad(
|
|
229
|
-
Cast(ExtractMonth("begin"), models.CharField()), 2, fill_text=Value("0")
|
|
230
|
-
),
|
|
304
|
+
Cast(ExtractYear(begin), models.CharField()),
|
|
231
305
|
LPad(
|
|
232
|
-
Cast(
|
|
306
|
+
Cast(ExtractMonth(begin), models.CharField()), 2, fill_text=Value("0")
|
|
233
307
|
),
|
|
308
|
+
LPad(Cast(ExtractDay(begin), models.CharField()), 2, fill_text=Value("0")),
|
|
234
309
|
]
|
|
235
310
|
if log_format:
|
|
236
311
|
parts.append(Value(f".{log_format.ext}"))
|
|
237
312
|
|
|
238
|
-
return self.annotate(**{field_name: Concat(*parts)})
|
|
313
|
+
return self.annotate(**{field_name: Concat(*parts, output_field=CharField())})
|
|
239
314
|
|
|
240
315
|
|
|
241
316
|
class ArchiveIndex(models.Model):
|
|
317
|
+
"""
|
|
318
|
+
The ArchiveIndex table stores references to serialized site log files indexed by the time
|
|
319
|
+
range in which they were current. The primary purpose of this table is to allow serialized
|
|
320
|
+
site log formats to change over time while maintaining a full historical record.
|
|
321
|
+
"""
|
|
322
|
+
|
|
242
323
|
site = models.ForeignKey(
|
|
243
324
|
"slm.Site", on_delete=models.CASCADE, null=False, related_name="indexes"
|
|
244
325
|
)
|
|
245
326
|
|
|
246
|
-
#
|
|
247
|
-
|
|
248
|
-
null=
|
|
249
|
-
db_index=True,
|
|
250
|
-
help_text=_("The point in time at which this archive became valid."),
|
|
327
|
+
# bounds are inclusive/exclusive
|
|
328
|
+
valid_range = DateTimeRangeField(
|
|
329
|
+
null=True, blank=True, default_bounds="[)", db_index=True
|
|
251
330
|
)
|
|
252
331
|
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
)
|
|
332
|
+
@property
|
|
333
|
+
def begin(self) -> datetime:
|
|
334
|
+
return getattr(self, "_begin", self.valid_range.lower)
|
|
335
|
+
|
|
336
|
+
@begin.setter
|
|
337
|
+
def begin(self, begin: datetime):
|
|
338
|
+
self._begin = begin
|
|
339
|
+
|
|
340
|
+
@property
|
|
341
|
+
def end(self) -> t.Optional[datetime]:
|
|
342
|
+
return getattr(self, "_end", self.valid_range.upper)
|
|
343
|
+
|
|
344
|
+
@end.setter
|
|
345
|
+
def end(self, end: datetime):
|
|
346
|
+
self._end = end
|
|
259
347
|
|
|
260
348
|
objects = ArchiveIndexManager.from_queryset(ArchiveIndexQuerySet)()
|
|
261
349
|
|
|
350
|
+
def clean(self):
|
|
351
|
+
super().clean()
|
|
352
|
+
|
|
353
|
+
# Skip validation if range is missing
|
|
354
|
+
if not self.valid_range or not self.site_id:
|
|
355
|
+
return
|
|
356
|
+
|
|
357
|
+
# Exclude self if updating
|
|
358
|
+
overlaps = ArchiveIndex.objects.filter(
|
|
359
|
+
site=self.site, valid_range__overlap=self.valid_range
|
|
360
|
+
)
|
|
361
|
+
if self.pk:
|
|
362
|
+
overlaps = overlaps.exclude(pk=self.pk)
|
|
363
|
+
|
|
364
|
+
if overlaps.exists():
|
|
365
|
+
overlapping_ranges = [f"[{o.begin}, {o.end or '∞'})" for o in overlaps]
|
|
366
|
+
message = _(
|
|
367
|
+
"This time range overlaps with an existing archive index for this site:\n"
|
|
368
|
+
) + "\n".join(overlapping_ranges)
|
|
369
|
+
raise ValidationError({"valid_range": message})
|
|
370
|
+
|
|
262
371
|
def __str__(self):
|
|
263
372
|
return (
|
|
264
373
|
f"{self.site.name} | {self.begin.date()} - "
|
|
265
|
-
f
|
|
374
|
+
f"{self.end.date() if self.end else 'present'}"
|
|
266
375
|
)
|
|
267
376
|
|
|
268
377
|
def delete(self, using=None, keep_parents=False):
|
|
@@ -273,12 +382,20 @@ class ArchiveIndex(models.Model):
|
|
|
273
382
|
return ArchiveIndex.objects.filter(pk=self.pk).delete()
|
|
274
383
|
|
|
275
384
|
class Meta:
|
|
276
|
-
ordering = ("-
|
|
385
|
+
ordering = ("-valid_range",)
|
|
277
386
|
indexes = [
|
|
278
|
-
models.Index(fields=("
|
|
279
|
-
|
|
387
|
+
models.Index(fields=("site", "valid_range")),
|
|
388
|
+
]
|
|
389
|
+
constraints = [
|
|
390
|
+
ExclusionConstraint(
|
|
391
|
+
name="no_overlapping_ranges_per_site",
|
|
392
|
+
expressions=[
|
|
393
|
+
("site", RangeOperators.EQUAL),
|
|
394
|
+
("valid_range", RangeOperators.OVERLAPS),
|
|
395
|
+
],
|
|
396
|
+
deferrable=Deferrable.DEFERRED,
|
|
397
|
+
)
|
|
280
398
|
]
|
|
281
|
-
unique_together = (("site", "begin"),)
|
|
282
399
|
verbose_name = "Archive Index"
|
|
283
400
|
verbose_name_plural = "Archive Index"
|
|
284
401
|
|
|
@@ -300,6 +417,9 @@ class ArchivedSiteLogManager(models.Manager):
|
|
|
300
417
|
return file
|
|
301
418
|
elif file:
|
|
302
419
|
file.delete()
|
|
420
|
+
filename = index.site.get_filename(
|
|
421
|
+
log_format=log_format, epoch=index.begin, lower_case=True
|
|
422
|
+
)
|
|
303
423
|
return self.model.objects.create(
|
|
304
424
|
site=index.site,
|
|
305
425
|
log_format=log_format,
|
|
@@ -307,14 +427,12 @@ class ArchivedSiteLogManager(models.Manager):
|
|
|
307
427
|
timestamp=index.begin,
|
|
308
428
|
mimetype=log_format.mimetype,
|
|
309
429
|
file_type=SLMFileType.SITE_LOG,
|
|
310
|
-
name=
|
|
430
|
+
name=filename,
|
|
311
431
|
file=ContentFile(
|
|
312
432
|
SiteLogSerializer(instance=index.site)
|
|
313
433
|
.format(log_format)
|
|
314
434
|
.encode("utf-8"),
|
|
315
|
-
name=
|
|
316
|
-
log_format=log_format, epoch=index.begin
|
|
317
|
-
),
|
|
435
|
+
name=filename,
|
|
318
436
|
),
|
|
319
437
|
gml_version=(
|
|
320
438
|
GeodesyMLVersion.latest()
|
|
@@ -333,7 +451,11 @@ class ArchivedSiteLogManager(models.Manager):
|
|
|
333
451
|
|
|
334
452
|
class ArchivedSiteLogQuerySet(models.QuerySet):
|
|
335
453
|
def annotate_filenames(
|
|
336
|
-
self,
|
|
454
|
+
self,
|
|
455
|
+
name_len: t.Optional[int] = None,
|
|
456
|
+
field_name: str = "filename",
|
|
457
|
+
lower_case: bool = False,
|
|
458
|
+
log_format: t.Optional[SiteLogFormat] = None,
|
|
337
459
|
):
|
|
338
460
|
"""
|
|
339
461
|
Add the log names (w/o) extension as a property called filename to
|
|
@@ -355,17 +477,21 @@ class ArchivedSiteLogQuerySet(models.QuerySet):
|
|
|
355
477
|
if lower_case:
|
|
356
478
|
name_str = Lower(name_str)
|
|
357
479
|
|
|
480
|
+
index_begin = Func(
|
|
481
|
+
F("index__valid_range"), function="lower", output_field=DateTimeField()
|
|
482
|
+
)
|
|
483
|
+
|
|
358
484
|
parts = [
|
|
359
485
|
name_str,
|
|
360
486
|
Value("_"),
|
|
361
|
-
Cast(ExtractYear(
|
|
487
|
+
Cast(ExtractYear(index_begin), models.CharField()),
|
|
362
488
|
LPad(
|
|
363
|
-
Cast(ExtractMonth(
|
|
489
|
+
Cast(ExtractMonth(index_begin), models.CharField()),
|
|
364
490
|
2,
|
|
365
491
|
fill_text=Value("0"),
|
|
366
492
|
),
|
|
367
493
|
LPad(
|
|
368
|
-
Cast(ExtractDay(
|
|
494
|
+
Cast(ExtractDay(index_begin), models.CharField()),
|
|
369
495
|
2,
|
|
370
496
|
fill_text=Value("0"),
|
|
371
497
|
),
|
|
@@ -382,7 +508,7 @@ class ArchivedSiteLog(SiteFile):
|
|
|
382
508
|
ArchiveIndex, on_delete=models.CASCADE, related_name="files"
|
|
383
509
|
)
|
|
384
510
|
|
|
385
|
-
name = models.CharField(max_length=50)
|
|
511
|
+
name = models.CharField(max_length=50, db_index=True)
|
|
386
512
|
|
|
387
513
|
objects = ArchivedSiteLogManager.from_queryset(ArchivedSiteLogQuerySet)()
|
|
388
514
|
|
|
@@ -391,6 +517,10 @@ class ArchivedSiteLog(SiteFile):
|
|
|
391
517
|
return f"[{self.index}] {self.name}"
|
|
392
518
|
return f"[{self.index}] {os.path.basename(self.file.path)}"
|
|
393
519
|
|
|
520
|
+
@cached_property
|
|
521
|
+
def link(self):
|
|
522
|
+
return reverse("slm_public_api:archive-detail", kwargs={"pk": self.pk})
|
|
523
|
+
|
|
394
524
|
def parse(self) -> BaseBinder:
|
|
395
525
|
if self.log_format is SiteLogFormat.GEODESY_ML:
|
|
396
526
|
return xsd_parsing.SiteLogBinder(
|
slm/models/sitelog.py
CHANGED
|
@@ -9,7 +9,7 @@ from django.contrib.auth import get_user_model
|
|
|
9
9
|
from django.contrib.auth.models import Permission
|
|
10
10
|
from django.contrib.gis.db import models as gis_models
|
|
11
11
|
from django.core.exceptions import FieldDoesNotExist, ValidationError
|
|
12
|
-
from django.core.validators import MaxValueValidator, MinValueValidator
|
|
12
|
+
from django.core.validators import MaxValueValidator, MinValueValidator
|
|
13
13
|
from django.db import models, transaction
|
|
14
14
|
from django.db.models import (
|
|
15
15
|
CheckConstraint,
|
|
@@ -53,6 +53,7 @@ from slm.defines import (
|
|
|
53
53
|
SiteLogStatus,
|
|
54
54
|
TectonicPlates,
|
|
55
55
|
)
|
|
56
|
+
from slm.models.fields import StationNameField
|
|
56
57
|
from slm.utils import date_to_str
|
|
57
58
|
from slm.validators import get_validators
|
|
58
59
|
|
|
@@ -83,9 +84,7 @@ class SubquerySum(Subquery):
|
|
|
83
84
|
output_field = models.IntegerField()
|
|
84
85
|
|
|
85
86
|
def __init__(self, *args, **kwargs):
|
|
86
|
-
self.template = (
|
|
87
|
-
f'(SELECT SUM({kwargs.pop("field")}) ' f"FROM (%(subquery)s) _sum)"
|
|
88
|
-
)
|
|
87
|
+
self.template = f"(SELECT SUM({kwargs.pop('field')}) FROM (%(subquery)s) _sum)"
|
|
89
88
|
super().__init__(*args, **kwargs)
|
|
90
89
|
|
|
91
90
|
|
|
@@ -118,7 +117,7 @@ class SiteQuerySet(models.QuerySet):
|
|
|
118
117
|
|
|
119
118
|
latest_archive = ArchivedSiteLog.objects.filter(
|
|
120
119
|
Q(site=OuterRef("pk")) & Q(log_format=log_format)
|
|
121
|
-
).order_by("-
|
|
120
|
+
).order_by("-index__valid_range")
|
|
122
121
|
size_field = f"{log_format.ext if prefix is None else prefix}_size"
|
|
123
122
|
file_field = f"{log_format.ext if prefix is None else prefix}_file"
|
|
124
123
|
return self.annotate(
|
|
@@ -651,17 +650,11 @@ class Site(models.Model):
|
|
|
651
650
|
|
|
652
651
|
objects = SiteManager.from_queryset(SiteQuerySet)()
|
|
653
652
|
|
|
654
|
-
name =
|
|
655
|
-
max_length=
|
|
653
|
+
name = StationNameField(
|
|
654
|
+
max_length=50,
|
|
656
655
|
unique=True,
|
|
657
|
-
help_text=_(
|
|
658
|
-
"This is the 9 Character station name (XXXXMRCCC) used in RINEX 3 "
|
|
659
|
-
"filenames Format: (XXXX - existing four character IGS station "
|
|
660
|
-
"name, M - Monument or marker number (0-9), R - Receiver number "
|
|
661
|
-
"(0-9), CCC - Three digit ISO 3166-1 country code)"
|
|
662
|
-
),
|
|
656
|
+
help_text=_("The name of the station."),
|
|
663
657
|
db_index=True,
|
|
664
|
-
validators=[RegexValidator(r"[\w]{4}[\d]{2}[\w]{3}")],
|
|
665
658
|
)
|
|
666
659
|
|
|
667
660
|
# todo can site exist without agency?
|
|
@@ -779,6 +772,7 @@ class Site(models.Model):
|
|
|
779
772
|
(default False)
|
|
780
773
|
:return: The filename including extension.
|
|
781
774
|
"""
|
|
775
|
+
# TODO - make this pluggable
|
|
782
776
|
if epoch is None:
|
|
783
777
|
epoch = self.last_publish or self.last_update or self.created
|
|
784
778
|
if name_len is None and log_format is SiteLogFormat.LEGACY:
|
|
@@ -890,15 +884,19 @@ class Site(models.Model):
|
|
|
890
884
|
return user.agencies.filter(pk__in=self.agencies.all()).count() > 0
|
|
891
885
|
return False
|
|
892
886
|
|
|
893
|
-
def update_status(
|
|
887
|
+
def update_status(
|
|
888
|
+
self, save=True, user=None, timestamp=None, first_publish=False, reverted=False
|
|
889
|
+
):
|
|
894
890
|
"""
|
|
895
891
|
Update the denormalized data that is too expensive to query on the
|
|
896
892
|
fly. This includes flag count, moderation status and DateTimes. Also
|
|
897
893
|
check for and delete any review requests if a publish was done.
|
|
898
894
|
|
|
899
|
-
:param save:
|
|
895
|
+
:param save: If true the site row will be saved to the database.
|
|
900
896
|
:param user: The user responsible for a status update check
|
|
901
897
|
:param timestamp: The time at which the status update is triggered
|
|
898
|
+
:param first_publish: True if this is the first time the site log is being published.
|
|
899
|
+
:param reverted: A boolean indicating if this update was triggered by a reversion or not.
|
|
902
900
|
:return:
|
|
903
901
|
"""
|
|
904
902
|
if not timestamp:
|
|
@@ -926,6 +924,9 @@ class Site(models.Model):
|
|
|
926
924
|
self.review_request.delete()
|
|
927
925
|
|
|
928
926
|
if save:
|
|
927
|
+
setattr( # ugly but we need to pass info to the post_save signal handler somehow
|
|
928
|
+
self, "_reverted", reverted
|
|
929
|
+
)
|
|
929
930
|
self.save()
|
|
930
931
|
|
|
931
932
|
def revert(self):
|
|
@@ -933,7 +934,7 @@ class Site(models.Model):
|
|
|
933
934
|
for section in self.sections():
|
|
934
935
|
reverted |= getattr(self, section.accessor).revert()
|
|
935
936
|
if reverted:
|
|
936
|
-
self.update_status()
|
|
937
|
+
self.update_status(reverted=reverted)
|
|
937
938
|
return reverted
|
|
938
939
|
|
|
939
940
|
def published(self, epoch=None):
|
|
@@ -1194,6 +1195,9 @@ class SiteSectionQueryset(gis_models.QuerySet):
|
|
|
1194
1195
|
def sort(self, reverse=False):
|
|
1195
1196
|
return self
|
|
1196
1197
|
|
|
1198
|
+
class Meta:
|
|
1199
|
+
order_by = ("name",)
|
|
1200
|
+
|
|
1197
1201
|
|
|
1198
1202
|
class SiteLocationManager(SiteSectionManager):
|
|
1199
1203
|
pass
|
|
@@ -1260,7 +1264,7 @@ class SiteSection(gis_models.Model):
|
|
|
1260
1264
|
).delete()[0]
|
|
1261
1265
|
)
|
|
1262
1266
|
if reverted:
|
|
1263
|
-
self.site.update_status()
|
|
1267
|
+
self.site.update_status(reverted=reverted)
|
|
1264
1268
|
return reverted
|
|
1265
1269
|
|
|
1266
1270
|
@property
|
|
@@ -1694,7 +1698,7 @@ class SiteSubSection(SiteSection):
|
|
|
1694
1698
|
& Q(subsection=self.subsection)
|
|
1695
1699
|
).update(is_deleted=False)
|
|
1696
1700
|
if reverted:
|
|
1697
|
-
self.site.update_status()
|
|
1701
|
+
self.site.update_status(reverted=reverted)
|
|
1698
1702
|
return reverted
|
|
1699
1703
|
|
|
1700
1704
|
@cached_property
|
|
@@ -2754,7 +2758,7 @@ class SiteAntenna(SiteSubSection):
|
|
|
2754
2758
|
blank=False,
|
|
2755
2759
|
verbose_name=_("Date Installed (UTC)"),
|
|
2756
2760
|
help_text=_(
|
|
2757
|
-
"Enter the date the receiver was installed.
|
|
2761
|
+
"Enter the date the receiver was installed. Format: (CCYY-MM-DDThh:mmZ)"
|
|
2758
2762
|
),
|
|
2759
2763
|
db_index=True,
|
|
2760
2764
|
)
|
|
@@ -2999,8 +3003,7 @@ class SiteFrequencyStandard(SiteSubSection):
|
|
|
2999
3003
|
def effective(self):
|
|
3000
3004
|
if self.effective_start and self.effective_end:
|
|
3001
3005
|
return (
|
|
3002
|
-
f"{date_to_str(self.effective_start)}/"
|
|
3003
|
-
f"{date_to_str(self.effective_end)}"
|
|
3006
|
+
f"{date_to_str(self.effective_start)}/{date_to_str(self.effective_end)}"
|
|
3004
3007
|
)
|
|
3005
3008
|
elif self.effective_start:
|
|
3006
3009
|
return f"{date_to_str(self.effective_start)}"
|
|
@@ -3115,8 +3118,7 @@ class SiteCollocation(SiteSubSection):
|
|
|
3115
3118
|
def effective(self):
|
|
3116
3119
|
if self.effective_start and self.effective_end:
|
|
3117
3120
|
return (
|
|
3118
|
-
f"{date_to_str(self.effective_start)}/"
|
|
3119
|
-
f"{date_to_str(self.effective_end)}"
|
|
3121
|
+
f"{date_to_str(self.effective_start)}/{date_to_str(self.effective_end)}"
|
|
3120
3122
|
)
|
|
3121
3123
|
elif self.effective_start:
|
|
3122
3124
|
return f"{date_to_str(self.effective_start)}"
|
|
@@ -3229,8 +3231,7 @@ class MeteorologicalInstrumentation(SiteSubSection):
|
|
|
3229
3231
|
def effective(self):
|
|
3230
3232
|
if self.effective_start and self.effective_end:
|
|
3231
3233
|
return (
|
|
3232
|
-
f"{date_to_str(self.effective_start)}/"
|
|
3233
|
-
f"{date_to_str(self.effective_end)}"
|
|
3234
|
+
f"{date_to_str(self.effective_start)}/{date_to_str(self.effective_end)}"
|
|
3234
3235
|
)
|
|
3235
3236
|
elif self.effective_start:
|
|
3236
3237
|
return f"{date_to_str(self.effective_start)}"
|
|
@@ -3293,7 +3294,7 @@ class MeteorologicalInstrumentation(SiteSubSection):
|
|
|
3293
3294
|
blank=False,
|
|
3294
3295
|
null=True,
|
|
3295
3296
|
help_text=_(
|
|
3296
|
-
"Enter the effective start date for the sensor.
|
|
3297
|
+
"Enter the effective start date for the sensor. Format: (CCYY-MM-DD)"
|
|
3297
3298
|
),
|
|
3298
3299
|
db_index=True,
|
|
3299
3300
|
)
|
|
@@ -3302,7 +3303,7 @@ class MeteorologicalInstrumentation(SiteSubSection):
|
|
|
3302
3303
|
blank=True,
|
|
3303
3304
|
default=None,
|
|
3304
3305
|
help_text=_(
|
|
3305
|
-
"Enter the effective end date for the sensor.
|
|
3306
|
+
"Enter the effective end date for the sensor. Format: (CCYY-MM-DD)"
|
|
3306
3307
|
),
|
|
3307
3308
|
db_index=True,
|
|
3308
3309
|
)
|
|
@@ -3404,8 +3405,7 @@ class SiteHumiditySensor(MeteorologicalInstrumentation):
|
|
|
3404
3405
|
max_length=50,
|
|
3405
3406
|
verbose_name=_("Aspiration"),
|
|
3406
3407
|
help_text=_(
|
|
3407
|
-
"Enter the aspiration type if known. "
|
|
3408
|
-
"Format: (UNASPIRATED/NATURAL/FAN/etc)"
|
|
3408
|
+
"Enter the aspiration type if known. Format: (UNASPIRATED/NATURAL/FAN/etc)"
|
|
3409
3409
|
),
|
|
3410
3410
|
db_index=True,
|
|
3411
3411
|
)
|
|
@@ -3546,8 +3546,7 @@ class SiteTemperatureSensor(MeteorologicalInstrumentation):
|
|
|
3546
3546
|
max_length=50,
|
|
3547
3547
|
verbose_name=_("Aspiration"),
|
|
3548
3548
|
help_text=_(
|
|
3549
|
-
"Enter the aspiration type if known. "
|
|
3550
|
-
"Format: (UNASPIRATED/NATURAL/FAN/etc)"
|
|
3549
|
+
"Enter the aspiration type if known. Format: (UNASPIRATED/NATURAL/FAN/etc)"
|
|
3551
3550
|
),
|
|
3552
3551
|
db_index=True,
|
|
3553
3552
|
)
|
|
@@ -3664,8 +3663,7 @@ class Condition(SiteSubSection):
|
|
|
3664
3663
|
def effective(self):
|
|
3665
3664
|
if self.effective_start and self.effective_end:
|
|
3666
3665
|
return (
|
|
3667
|
-
f"{date_to_str(self.effective_start)}/"
|
|
3668
|
-
f"{date_to_str(self.effective_end)}"
|
|
3666
|
+
f"{date_to_str(self.effective_start)}/{date_to_str(self.effective_end)}"
|
|
3669
3667
|
)
|
|
3670
3668
|
elif self.effective_start:
|
|
3671
3669
|
return f"{date_to_str(self.effective_start)}"
|
|
@@ -3684,7 +3682,7 @@ class Condition(SiteSubSection):
|
|
|
3684
3682
|
null=True,
|
|
3685
3683
|
default=None,
|
|
3686
3684
|
help_text=_(
|
|
3687
|
-
"Enter the effective start date for the condition.
|
|
3685
|
+
"Enter the effective start date for the condition. Format: (CCYY-MM-DD)"
|
|
3688
3686
|
),
|
|
3689
3687
|
db_index=True,
|
|
3690
3688
|
)
|
|
@@ -3694,7 +3692,7 @@ class Condition(SiteSubSection):
|
|
|
3694
3692
|
null=True,
|
|
3695
3693
|
default=None,
|
|
3696
3694
|
help_text=_(
|
|
3697
|
-
"Enter the effective end date for the condition.
|
|
3695
|
+
"Enter the effective end date for the condition. Format: (CCYY-MM-DD)"
|
|
3698
3696
|
),
|
|
3699
3697
|
db_index=True,
|
|
3700
3698
|
)
|
|
@@ -3859,8 +3857,7 @@ class SiteLocalEpisodicEffects(SiteSubSection):
|
|
|
3859
3857
|
def effective(self):
|
|
3860
3858
|
if self.effective_start and self.effective_end:
|
|
3861
3859
|
return (
|
|
3862
|
-
f"{date_to_str(self.effective_start)}/"
|
|
3863
|
-
f"{date_to_str(self.effective_end)}"
|
|
3860
|
+
f"{date_to_str(self.effective_start)}/{date_to_str(self.effective_end)}"
|
|
3864
3861
|
)
|
|
3865
3862
|
elif self.effective_start:
|
|
3866
3863
|
return f"{date_to_str(self.effective_start)}"
|