igs-slm 0.1.4b0__py3-none-any.whl → 0.1.5b1__py3-none-any.whl

slm/migrations/0032_archiveindex_valid_range_and_more.py

@@ -0,0 +1,84 @@
+# Generated by Django 4.2.20 on 2025-05-08 19:52
+
+import django.contrib.postgres.constraints
+import django.contrib.postgres.fields.ranges
+from django.contrib.postgres.fields.ranges import DateTimeTZRange
+from django.db import migrations, models
+
+
+def populate_valid_range(apps, schema_editor):
+    ArchiveIndex = apps.get_model("slm", "ArchiveIndex")
+
+    for row in ArchiveIndex.objects.all():
+        ArchiveIndex.objects.filter(pk=row.pk).update(
+            valid_range=DateTimeTZRange(row.begin, row.end)
+        )
+
+
+def reverse_valid_range(apps, schema_editor):
+    ArchiveIndex = apps.get_model("slm", "ArchiveIndex")
+
+    for row in ArchiveIndex.objects.all():
+        begin = row.valid_range.lower if row.valid_range else None
+        end = row.valid_range.upper if row.valid_range else None
+        ArchiveIndex.objects.filter(pk=row.pk).update(begin=begin, end=end)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("slm", "0031_alter_antenna_features"),
+    ]
+
+    operations = [
+        migrations.RunSQL("CREATE EXTENSION IF NOT EXISTS citext;"),
+        migrations.RunSQL("CREATE EXTENSION IF NOT EXISTS btree_gist;"),
+        migrations.AddField(
+            model_name="archiveindex",
+            name="valid_range",
+            field=django.contrib.postgres.fields.ranges.DateTimeRangeField(
+                blank=True, db_index=True, null=True
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="archiveindex",
+            index=models.Index(
+                fields=["site", "valid_range"], name="slm_archive_site_id_5a8c6e_idx"
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="archiveindex",
+            constraint=django.contrib.postgres.constraints.ExclusionConstraint(
+                expressions=[("site", "="), ("valid_range", "&&")],
+                name="no_overlapping_ranges_per_site",
+            ),
+        ),
+        migrations.RunPython(populate_valid_range, reverse_code=reverse_valid_range),
+        migrations.AlterModelOptions(
+            name="archiveindex",
+            options={
+                "ordering": ("-valid_range",),
+                "verbose_name": "Archive Index",
+                "verbose_name_plural": "Archive Index",
+            },
+        ),
+        migrations.RemoveIndex(
+            model_name="archiveindex",
+            name="slm_archive_begin_b89d03_idx",
+        ),
+        migrations.RemoveIndex(
+            model_name="archiveindex",
+            name="slm_archive_site_id_c1b6a6_idx",
+        ),
+        migrations.AlterUniqueTogether(
+            name="archiveindex",
+            unique_together=set(),
+        ),
+        migrations.RemoveField(
+            model_name="archiveindex",
+            name="begin",
+        ),
+        migrations.RemoveField(
+            model_name="archiveindex",
+            name="end",
+        ),
+    ]

slm/migrations/add_index_order_index.py

@@ -0,0 +1,54 @@
+from django.db import migrations
+
+# TODO when migrations are squashed - we need to reinclude this one
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        # Replace with your latest migration
+        ("slm", "0032_archiveindex_valid_range_and_more"),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            sql="""
+                CREATE INDEX IF NOT EXISTS idx_archiveindex_lower_bound
+                ON slm_archiveindex (lower(valid_range));
+            """,
+            reverse_sql="""
+                DROP INDEX IF EXISTS idx_archiveindex_lower_bound;
+            """,
+        ),
+        migrations.RunSQL(
+            sql="""
+                CREATE INDEX IF NOT EXISTS idx_archiveindex_upper_bound
+                ON slm_archiveindex (upper(valid_range));
+            """,
+            reverse_sql="""
+                DROP INDEX IF EXISTS idx_archiveindex_upper_bound;
+            """,
+        ),
+        # for idempotency - drop it first
+        migrations.RunSQL(
+            """
+            ALTER TABLE slm_archiveindex
+            DROP CONSTRAINT IF EXISTS index_valid_range_lower_not_null;
+            """,
+            reverse_sql="""
+            ALTER TABLE slm_archiveindex
+            ADD CONSTRAINT index_valid_range_lower_not_null
+            CHECK (lower(valid_range) IS NOT NULL);
+            """,
+        ),
+        migrations.RunSQL(
+            """
+            ALTER TABLE slm_archiveindex
+            ADD CONSTRAINT index_valid_range_lower_not_null
+            CHECK (lower(valid_range) IS NOT NULL);
+            """,
+            reverse_sql="""
+            ALTER TABLE slm_archiveindex
+            DROP CONSTRAINT IF EXISTS index_valid_range_lower_not_null;
+            """,
+        ),
+    ]

slm/migrations/normalize_index.py

@@ -0,0 +1,147 @@
+import os
+from pathlib import Path
+
+from django.db import migrations
+from django.db.models import DateTimeField
+from django.db.models.functions import Lower
+
+
+def site_upload_path(instance, filename):
+    """
+    The historical site_upload_path implementation.
+    """
+    return f"archive/{instance.site.name}/{filename}"
+
+
+def normalize_index(apps, schema_editor):
+    from django.conf import settings
+
+    ArchivedSiteLog = apps.get_model("slm", "ArchivedSiteLog")
+    Site = apps.get_model("slm", "Site")
+
+    # paths to update - we do this after the database updates succeed, so if they're rolled
+    # back after an exception the disk state will still match the db
+    path_changes = []
+    unindexed = absolute = normalized = timestamps = 0
+
+    for archived in ArchivedSiteLog.objects.filter(file__startswith="/"):
+        current_name = Path(archived.file.name)
+        # change to relative path
+        archived.file.name = site_upload_path(archived, current_name.name)
+        assert not archived.file.name.startswith("/")
+        if not Path(archived.file.path).is_file():
+            if current_name.is_file():
+                raise RuntimeError(
+                    f"Archived file {archived.file.name} does not exist at the new path: {Path(archived.file.path)}"
+                )
+            else:
+                print(
+                    f"WARNING: Archived file {archived.file.name} does not exist on disk!"
+                )
+        archived.name = Path(archived.file.name).name
+        archived.save(update_fields=["file", "name"])
+        absolute += 1
+        # print(f"Fixed indexed file {current_name} -> {archived.file.name}")
+
+    # remove unindexed files
+    for site in Site.objects.all():
+        site_dir = Path(settings.MEDIA_ROOT / "archive" / site.name)
+        if not site_dir.is_dir():
+            continue
+        files = {site_dir / file for file in os.listdir(site_dir)}
+        if ArchivedSiteLog.objects.filter(site=site).count() != len(files):
+            for archived in ArchivedSiteLog.objects.filter(site=site):
+                try:
+                    files.remove(Path(archived.file.path))
+                except KeyError as err:
+                    raise RuntimeError() from err  # this should not happen
+            for file in files:
+                # print(f"WARNING: deleting unindexed {file} in archive!")
+                unindexed += 1
+                file.unlink()
+
+    names = set()
+    for archived in ArchivedSiteLog.objects.annotate(
+        begin=Lower("index__valid_range", output_field=DateTimeField())
+    ).all():
+        current_path = Path(archived.file.path)
+        new_path = Path(archived.file.path)
+        update_fields = []
+
+        if archived.timestamp != archived.begin:
+            update_fields.append("timestamp")
+            archived.timestamp = archived.begin
+            timestamps += 1
+
+        # make lowercase if necessary
+        if not current_path.name.islower():
+            new_path = current_path.with_name(current_path.name.lower())
+
+        # convert nine to four char if necessary
+        if (
+            archived.log_format == 1
+            and "_" in new_path.name
+            and new_path.name.index("_") == 9
+        ):
+            new_path = new_path.with_name(f"{new_path.name[0:4]}{new_path.name[9:]}")
+        # convert four char to nine if necessary
+        elif (
+            archived.log_format == 4
+            and "_" in new_path.name
+            and new_path.name.index("_") == 4
+        ):
+            new_path = new_path.with_name(
+                f"{archived.index.site.name[0:9].lower()}{new_path.name[4:]}"
+            )
+
+        # add timestamp if necessary
+        # collisions - can happen in weird 4char/9char inconsistent same date cases
+        if new_path.name.count("_") == 2 or new_path.name in names:
+            new_path = new_path.with_name(
+                f"{new_path.stem[0 : new_path.stem.rindex('_')]}_{archived.index.valid_range.lower.strftime('%H%M%S')}{new_path.suffix}"
+            )
+
+        if new_path != current_path:
+            normalized += 1
+            path_changes.append((current_path, new_path))
+            names.add(new_path.name)
+            archived.file.name = archived.file.name.replace(
+                current_path.name, new_path.name
+            )
+            archived.name = Path(archived.file.name).name
+            update_fields.extend(["file", "name"])
+            # print(f"Normalized indexed file name {current_path.name} -> {new_path.name}")
+        else:
+            names.add(current_path.name)
+
+        if update_fields:
+            archived.save(update_fields=update_fields)
+
+    for old, new in path_changes:
+        if (
+            new.is_file()
+            and not (new.name == old.name.lower())
+            and new.read_text() != old.read_text()
+        ):
+            print(f"WARNING: {old} will overwrite {new}!")
+        old.rename(new)
+
+    print(f"\nConverted {absolute} absolute paths to relative paths.")
+    print(f"Deleted {unindexed} files.")
+    print(f"Normalized {normalized} files.")
+    print(f"Synchronized {timestamps} file timestamps.")
+
+
+def noop(apps, schema_editor):
+    pass
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        # Replace with your latest migration
+        ("slm", "add_index_order_index"),
+    ]
+
+    operations = [
+        migrations.RunPython(normalize_index, reverse_code=noop, atomic=True),
+    ]

slm/migrations/simplify_index.py

@@ -0,0 +1,48 @@
+from pathlib import Path
+
+from django.db import migrations
+
+
+def noop(apps, schema_editor):
+    pass
+
+
+def simplify_files(apps, schema_editor):
+    ArchivedSiteLog = apps.get_model("slm", "ArchivedSiteLog")
+
+    # paths to update - we do this after the database updates succeed, so if they're rolled
+    # back after an exception the disk state will still match the db
+    path_changes = []
+    simplified = 0
+
+    for archived in ArchivedSiteLog.objects.order_by("timestamp"):
+        if archived.file.name.count("_") == 2:
+            # make sure the first on a date does not have the timestamp on the end of it
+            current_path = Path(archived.file.path)
+            stem, ext = current_path.stem, current_path.suffix
+            simple_path = current_path.with_name(f"{stem[: stem.rindex('_')]}{ext}")
+            if not simple_path.exists():
+                path_changes.append((current_path, simple_path))
+                archived.file.name = archived.file.name.replace(
+                    current_path.name, simple_path.name
+                )
+                archived.name = Path(archived.file.name).name
+                archived.save(update_fields=["file", "name"])
+                # print(f"Simplified indexed file name {current_path.name} -> {simple_path.name}")
+                simplified += 1
+
+    for old, new in path_changes:
+        old.rename(new)
+
+    print(f"\nSimplified {simplified} files.")
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        # Replace with your latest migration
+        ("slm", "normalize_index"),
+    ]
+
+    operations = [
+        migrations.RunPython(simplify_files, reverse_code=noop, atomic=True),
+    ]

slm/migrations/verify_index.py

@@ -0,0 +1,67 @@
+from pathlib import Path
+from pprint import pformat
+
+from django.db import migrations
+from django.db.models import Count, Func, Q
+
+
+def noop(apps, schema_editor):
+    pass
+
+
+def verify_index(apps, schema_editor):
+    """
+    Run final sanity check.
+
+    Verifies the following conditions:
+
+        1. All files exist
+        2. All db row names match the file names
+        3. No files are referenced by more than one db row
+        4. That all file timestamps match the index begin timestamp
+    """
+    ArchivedSiteLog = apps.get_model("slm", "ArchivedSiteLog")
+
+    assert ArchivedSiteLog.objects.filter(file__startswith="/").count() == 0, (
+        "There are still absolute paths in the archived site log file field."
+    )
+
+    # final sanity checks
+    # check that all files exist
+    for archived in ArchivedSiteLog.objects.all():
+        assert Path(archived.file.path).is_file(), (
+            f"WARNING: {archived} index file does not exist!"
+        )
+        assert archived.name == Path(archived.file.name).name, (
+            f"{Path(archived.file.name).name}/{archived.name} mismatch!"
+        )
+
+    # check that all files are unique
+    duplicates = (
+        ArchivedSiteLog.objects.values("file")  # file.name
+        .annotate(count=Count("id"))
+        .filter(count__gt=1)
+    )
+
+    # this must be true because following migrations will make this field unique!
+    assert not duplicates.exists(), (
+        f"Some indexes share the same files: {pformat(duplicates)}"
+    )
+
+    bad_timestamps = ArchivedSiteLog.objects.filter(
+        ~Q(timestamp=Func("index__valid_range", function="lower"))
+    )
+    assert not bad_timestamps.count(), (
+        f"Some indexes timestamps do not align with the valid range: {pformat(bad_timestamps)}"
+    )
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        # Replace with your latest migration
+        ("slm", "simplify_index"),
+    ]
+
+    operations = [
+        migrations.RunPython(verify_index, reverse_code=noop),
+    ]

slm/models/__init__.py

@@ -55,6 +55,7 @@ from slm.models.system import (
     SiteFile,
     SiteFileUpload,
     SiteTideGauge,
+    SLMVersion,
     TideGauge,
 )
 from slm.models.user import User, UserProfile
@@ -115,4 +116,5 @@ __all__ = [
     "UserProfile",
     "About",
     "Help",
+    "SLMVersion",
 ]

slm/models/alerts.py

@@ -315,7 +315,7 @@ class Alert(PolymorphicModel):
         if self.site_alert:
             return reverse("slm:edit", kwargs={"station": self.target.name})
         elif self.agency_alert:
-            return f'{reverse("slm:home")}?agency={self.target.pk}'
+            return f"{reverse('slm:home')}?agency={self.target.pk}"
         elif self.user_alert:
             return f"mailto:{self.target.email}"
         return reverse("slm:alert", kwargs={"alert": self.pk})
@@ -398,7 +398,7 @@ class Alert(PolymorphicModel):
         default=False,
         blank=True,
         help_text=_(
-            "Do not allow target users to clear this alert, only admins may " "clear."
+            "Do not allow target users to clear this alert, only admins may clear."
         ),
     )
 
@@ -426,7 +426,7 @@ class Alert(PolymorphicModel):
         null=False,
         blank=False,
         help_text=_(
-            "If true, an email will be sent for this alert to every targeted " "user."
+            "If true, an email will be sent for this alert to every targeted user."
         ),
     )
 
@@ -792,8 +792,7 @@ class GeodesyMLInvalid(AutomatedAlertMixin, SiteFile, Alert):
     published = models.BooleanField(
         null=False,
         help_text=_(
-            "True if this alert was issued from the published version of the "
-            "site log."
+            "True if this alert was issued from the published version of the site log."
         ),
     )
 
@@ -908,8 +907,7 @@ class ReviewRequested(AutomatedAlertMixin, Alert):
                 ).format(self.requester.email, self.requester.name)
                 if self.requester
                 else _(
-                    "A request has been made to publish the updates to this "
-                    "site log."
+                    "A request has been made to publish the updates to this site log."
                 )
             )
         super().save(*args, **kwargs)
@@ -1094,10 +1092,9 @@ class SiteLogPublished(AutomatedAlertMixin, Alert):
                 "An updated log has been published for this site. Download "
                 "the new {legacy_file} or the new {geodesyml_file}."
             ).format(
-                legacy_file=f'<a href="{legacy_link}" download>'
-                f'{_("legacy file")}</a>',
+                legacy_file=f'<a href="{legacy_link}" download>{_("legacy file")}</a>',
                 geodesyml_file=f'<a href="{gml_link}" download>'
-                f'{_("GeodesyML file")}</a>',
+                f"{_('GeodesyML file')}</a>",
             )
         super().save(*args, **kwargs)
 

slm/models/data.py

@@ -15,8 +15,7 @@ class DataAvailability(models.Model):
 
     def __str__(self):
         return (
-            f"[{self.site}] ({self.rinex_version.label}) "
-            f"{self.rate.label} {self.last}"
+            f"[{self.site}] ({self.rinex_version.label}) {self.rate.label} {self.last}"
         )
 
     class Meta:

slm/models/equipment.py

@@ -141,7 +141,7 @@ class Antenna(Equipment):
     def full(self):
         return (
             f"{self.model} "
-            f'{self.reference_point.label if self.reference_point else ""} '
+            f"{self.reference_point.label if self.reference_point else ''} "
             f"{self.features.label}"
         )
 

slm/models/fields.py

@@ -0,0 +1,41 @@
+# upstream/fields.py
+from django.conf import settings
+from django.core.validators import RegexValidator
+from django.db import models
+
+
+class StationNameField(models.CharField):
+    """
+    The StationNameField allows validation and help text to be configured in settings.
+    These values will not be deconstructed into the migration files allowing users of
+    the SLM to define their own semantics and validation logic around station names.
+    """
+
+    class StationNameValidator(RegexValidator):
+        pass
+
+    def __init__(self, *args, **kwargs):
+        if regex := getattr(settings, "SLM_STATION_NAME_REGEX", None):
+            kwargs.setdefault("validators", [])
+            kwargs["validators"].append(self.StationNameValidator(regex))
+        self._help_text = kwargs.pop("help_text", None)
+        help_text = getattr(settings, "SLM_STATION_NAME_HELP", self._help_text)
+        if help_text:
+            kwargs["help_text"] = help_text
+        super().__init__(*args, **kwargs)
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+        # Ensure the regex pattern doesn't get serialized if it came from settings
+        validators = [
+            validator
+            for validator in kwargs.pop("validators", [])
+            if not isinstance(validator, self.StationNameValidator)
+        ]
+        if validators:
+            kwargs["validators"] = validators
+
+        # we want the cannonical help text to be used in migration files
+        if self._help_text is not None:
+            kwargs["help_text"] = self._help_text
+        return name, path, args, kwargs