igs-slm 0.1.0b0__py3-none-any.whl

slm/management/commands/update_data_availability.py

@@ -0,0 +1,167 @@
+"""
+Update the data availability information for each station.
+"""
+
+import typing as t
+from datetime import datetime, timedelta
+
+from django.utils.timezone import now
+from django.utils.translation import gettext as _
+from django_typer.management import TyperCommand, model_parser_completer
+from typer import Argument, Option
+from typing_extensions import Annotated
+
+from igs_tools.defines import DataCenter
+from igs_tools.directory import DirectoryListing
+from slm.defines import DataRate, RinexVersion
+from slm.management.commands import EnumParser
+from slm.models import DataAvailability, Site
+
+
+class Command(TyperCommand):
+    help = _("Update the data availability information for each station.")
+
+    LOOKBACK = 60  # 60 days
+
+    UNHEALTHY = (now() - timedelta(days=30)).date()
+    STALE = (now() - timedelta(days=10)).date()
+
+    suppressed_base_arguments = {
+        *TyperCommand.suppressed_base_arguments,
+        "version",
+        "pythonpath",
+        "settings",
+        "skip-checks",
+    }
+    requires_migrations_checks = False
+    requires_system_checks = []
+
+    def handle(
+        self,
+        sites: Annotated[
+            t.Optional[t.List[Site]],
+            Argument(
+                **model_parser_completer(
+                    Site, lookup_field="name", case_insensitive=True
+                ),
+                help=_(
+                    "The station(s) to update, if unspecified, update data "
+                    "availability information for all of them."
+                ),
+            ),
+        ] = None,
+        username: Annotated[str, Option(help=_("Username to login to ftp."))] = "",
+        password: Annotated[
+            str,
+            Option(
+                prompt=True,
+                prompt_required=False,
+                hide_input=True,
+                help=_("Password to login to ftp."),
+            ),
+        ] = "",
+        lookback: Annotated[
+            int,
+            Option(
+                "-l", "--lookback", help=_("The number of days into the past to look.")
+            ),
+        ] = LOOKBACK,
+        data_rates: Annotated[
+            t.Optional[t.List[DataRate]],
+            Option(
+                "--rate",
+                parser=EnumParser(DataRate, field="slug"),
+                help=_(
+                    "Restrict the data rates to update availability for. {rates}"
+                ).format(rates=", ".join([rate.slug for rate in DataRate])),
+            ),
+        ] = [rate.slug for rate in DirectoryListing.data_rates],
+        rinex_versions: Annotated[
+            t.Optional[t.List[RinexVersion]],
+            Option(
+                "--rinex",
+                parser=EnumParser(RinexVersion, field="slug"),
+                help=_(
+                    "Restrict rinex versions to update availability for ({versions})."
+                ).format(
+                    versions=", ".join(
+                        [ver.slug for ver in RinexVersion.major_versions()]
+                    )
+                ),
+            ),
+        ] = [ver.slug for ver in DirectoryListing.rinex_versions],
+        data_centers: Annotated[
+            t.Optional[t.List[DataCenter]],
+            Option(
+                "--center",
+                parser=EnumParser(DataCenter, field="label"),
+                help=_(
+                    "Restrict data centers to update availability for ({centers})."
+                ).format(centers=", ".join([str(dc) for dc in DataCenter])),
+            ),
+        ] = [dc for dc in DirectoryListing.data_centers],
+    ):
+        availability = {}
+        unrecognized = set()
+        for listing in DirectoryListing(
+            stations=[site.name for site in sites or Site.objects.active()],
+            start=now(),
+            end=now() - timedelta(days=lookback),
+            username=username,
+            password=password,
+            data_rates=data_rates or [],
+            rinex_versions=rinex_versions or [],
+            data_centers=data_centers or [],
+        ):
+            if listing.file_type not in {"d", "O"}:
+                continue
+            try:
+                avail, created = DataAvailability.objects.get_or_create(
+                    site=Site.objects.get(name__istartswith=listing.station),
+                    rinex_version=listing.rinex_version,
+                    rate=listing.data_rate,
+                    last=(
+                        listing.date.date()
+                        if isinstance(listing.date, datetime)
+                        else listing.date
+                    ),
+                )
+            except Site.DoesNotExist:
+                unrecognized.add(listing.station)
+                continue
+            availability.setdefault(avail.site.name, avail.last)
+            if availability[avail.site.name] < avail.last:
+                availability[avail.site.name] = avail.last
+
+        unhealthy = []
+        stale = []
+        healthy = []
+        for station, last in availability.items():
+            if last < self.UNHEALTHY:
+                unhealthy.append((station, last))
+            elif last < self.STALE:
+                stale.append((station, last))
+            else:
+                healthy.append((station, last))
+
+        for station, last in healthy:
+            self.stdout.write(self.style.SUCCESS(f"{station} {last}"))
+        for station, last in stale:
+            self.stdout.write(self.style.WARNING(f"{station} {last}"))
+        for station, last in unhealthy:
+            self.stdout.write(self.style.ERROR(f"{station} {last}"))
+
+        self.stdout.write(
+            self.style.WARNING(
+                _("Ignored {unrecognized} stations.").format(
+                    unrecognized=len(unrecognized)
+                )
+            )
+        )
+        self.stdout.write(
+            self.style.SUCCESS(
+                _("Updated {availability} stations.").format(
+                    availability=len(availability)
+                )
+            )
+        )

slm/management/commands/validate_db.py

@@ -0,0 +1,186 @@
+"""
+The SLM supports pluggable validation logic for all site log sections and fields. Since
+this logic is configuraton specific, it may change, diverging from the validation flags
+recorded against the previous validation configuration in the database.
+
+Run this command to run all validation routines against all current sitelog fields
+in the database.
+"""
+
+import typing as t
+
+from django.core.exceptions import ValidationError
+from django.db import transaction
+from django.db.models import Q, Sum
+from django.utils.translation import gettext as _
+from django_typer.management import TyperCommand, model_parser_completer
+from tqdm import tqdm
+from typer import Argument, Option
+from typing_extensions import Annotated
+
+from slm.defines import SiteLogStatus
+from slm.models import GeodesyMLInvalid, Site
+
+
+class Command(TyperCommand):
+    help = _(
+        "Validate and update status of head state of all existing site logs. "
+        "This might be necessary to run if the validation configuration is "
+        "updated."
+    )
+
+    suppressed_base_arguments = {
+        *TyperCommand.suppressed_base_arguments,
+        "version",
+        "pythonpath",
+        "settings",
+    }
+
+    def handle(
+        self,
+        sites: Annotated[
+            t.Optional[t.List[Site]],
+            Argument(
+                **model_parser_completer(
+                    Site, lookup_field="name", case_insensitive=True
+                ),
+                help=_("The name of the site(s). Default - all sites."),
+            ),
+        ] = None,
+        clear: Annotated[
+            bool, Option("--clear", help=_("Clear existing validation flags."))
+        ] = False,
+        schema: Annotated[
+            bool,
+            Option(
+                "--schema",
+                help=_(
+                    "Also perform validation of generated GeodesyML files "
+                    "against the latest schema."
+                ),
+            ),
+        ] = False,
+        all: Annotated[
+            bool,
+            Option(
+                "--all",
+                help=_("Validate all sites, not just the currently public sites."),
+            ),
+        ] = False,
+    ):
+        from slm.validators import set_bypass
+
+        set_bypass(True)
+        invalid = 0
+        valid = 0
+        critical = 0
+        old_flag_count = Site.objects.aggregate(Sum("num_flags"))["num_flags__sum"]
+        with transaction.atomic():
+            sites = sites or Site.objects.all()
+            if not all:
+                sites = Site.objects.filter(
+                    Q(pk__in=[site.pk for site in sites])
+                    & Q(status__in=[SiteLogStatus.PUBLISHED, SiteLogStatus.UPDATED])
+                )
+
+            with tqdm(
+                total=sites.count(),
+                desc=_("Validating"),
+                unit="sites",
+                postfix={"site": ""},
+            ) as p_bar:
+                for site in sites:
+                    p_bar.set_postfix({"site": site.name})
+                    for section in Site.sections():
+                        head = getattr(site, section.accessor).head()
+                        if not hasattr(head, "__iter__"):
+                            if head:
+                                head = [head]
+                            else:
+                                head = []
+                        for obj in head:
+                            if clear:
+                                obj._flags = {}
+                            try:
+                                obj.clean()
+                                obj.save()
+                            except ValidationError as verr:
+                                self.stderr.write(
+                                    self.style.ERROR(
+                                        _(
+                                            "Section {} of site {} has "
+                                            "critical error(s): {}"
+                                        ).format(
+                                            obj.__class__.__name__.lstrip("Site"),
+                                            site.name,
+                                            str(verr),
+                                        )
+                                    )
+                                )
+                                critical += 1
+                                # this type of error would normally block the
+                                # section from being saved - but we let this
+                                # go through here and record the error as a
+                                # normal flag
+                                for field, errors in verr.error_dict.items():
+                                    obj._flags[field] = "\n".join(
+                                        err.message for err in errors
+                                    )
+
+                    if schema:
+                        alert = GeodesyMLInvalid.objects.check_site(site=site)
+                        if alert:
+                            invalid += 1
+                        else:
+                            valid += 1
+
+                    # site.update_status(save=True)
+                    p_bar.update(n=1)
+
+            Site.objects.synchronize_denormalized_state(skip_form_updates=True)
+
+        if schema:
+            if valid:
+                self.stdout.write(
+                    self.style.SUCCESS(
+                        _("{valid} sites had valid GeodesyML documents.").format(
+                            valid=valid
+                        )
+                    )
+                )
+            if invalid:
+                self.stdout.write(
+                    self.style.ERROR(
+                        _(
+                            "{invalid} sites do not have valid GeodesyML documents."
+                        ).format(invalid=invalid)
+                    )
+                )
+
+        new_flags = Site.objects.aggregate(Sum("num_flags"))["num_flags__sum"]
+
+        delta = new_flags - old_flag_count
+
+        if delta >= 0:
+            change = "added"
+        else:
+            change = "removed"
+
+        self.stdout.write(
+            self.style.NOTICE(
+                _("{delta} flags were {change}.").format(
+                    delta=abs(delta), change=change
+                )
+            )
+        )
+
+        self.stdout.write(
+            self.style.NOTICE(
+                _(
+                    "There are a total of {new_flags} validation flags across "
+                    "{site_count} sites. {critical} are critical."
+                ).format(
+                    new_flags=new_flags, site_count=sites.count(), critical=critical
+                )
+            )
+        )

slm/management/commands/validate_gml.py

@@ -0,0 +1,73 @@
+"""
+Update the data availability information for each station.
+"""
+
+import typing as t
+from pathlib import Path
+
+from django.core.management import CommandError
+from django.utils.translation import gettext as _
+from django_typer.management import TyperCommand
+from lxml import etree
+from typer import Argument, Option
+from typing_extensions import Annotated
+
+from slm.defines import GeodesyMLVersion
+from slm.management.commands import EnumParser
+
+
+class Command(TyperCommand):
+    help = _("Validate the given xml document against the GeodesyML schema.")
+
+    suppressed_base_arguments = {
+        *TyperCommand.suppressed_base_arguments,
+        #    "version",
+        "pythonpath",
+        "settings",
+        "skip-checks",
+    }
+    requires_migrations_checks = False
+    requires_system_checks = []
+
+    def handle(
+        self,
+        file: Annotated[Path, Argument(help=_("Path to the GML file to validate."))],
+        version: Annotated[
+            t.Optional[GeodesyMLVersion],
+            Option(
+                "--version",
+                metavar="VERSION",
+                parser=EnumParser(GeodesyMLVersion, "version"),
+                help=_("The Geodesy ML version. ({versions})").format(
+                    versions=", ".join([str(gml.version) for gml in GeodesyMLVersion])
+                ),
+            ),
+        ] = None,
+    ):
+        if not file.exists():
+            raise CommandError(_("{file} does not exist!").format(file=file))
+        doc = etree.parse(file)
+        root = doc.getroot()
+
+        if not version:
+            version = GeodesyMLVersion(
+                root.nsmap.get(root.prefix, GeodesyMLVersion.latest())
+            )
+
+        self.stdout.write(_("Validating against: {version}").format(version=version))
+
+        result = version.schema.validate(doc)
+
+        if not result:
+            for error in version.schema.error_log:
+                self.stdout.write(
+                    self.style.ERROR(
+                        _("[{line}] {message}").format(
+                            line=error.line, message=error.message
+                        )
+                    )
+                )
+        else:
+            self.stdout.write(
+                self.style.SUCCESS(_("{file} is valid!").format(file=file))
+            )

slm/map/__init__.py

@@ -0,0 +1 @@
+default_app_config = "slm.map.apps.MapConfig"

slm/map/admin.py

@@ -0,0 +1,5 @@
+from django.contrib import admin
+
+from slm.map.models import MapSettings
+
+admin.site.register(MapSettings)

slm/map/api/edit/serializers.py

@@ -0,0 +1,28 @@
+from rest_framework import serializers
+
+from slm.api.edit import serializers as slm_serializers
+
+
+class StationSerializer(slm_serializers.StationSerializer):
+    llh = serializers.SerializerMethodField(read_only=True)
+
+    def get_llh(self, obj):
+        if hasattr(obj, "llh") and obj.llh:
+            return obj.llh[0], obj.llh[1], obj.llh[2]
+        return None
+
+    class Meta(slm_serializers.StationSerializer.Meta):
+        fields = (*slm_serializers.StationSerializer.Meta.fields, "llh")
+
+
+class StationMapSerializer(serializers.Serializer):
+    def to_representation(self, instance):
+        return {
+            "id": instance.pk,
+            "type": "Feature",
+            "geometry": {
+                "type": "Point",
+                "coordinates": [instance.llh[1], instance.llh[0]],
+            },
+            "properties": {"name": instance.name, "status": instance.status},
+        }

slm/map/api/edit/views.py

@@ -0,0 +1,46 @@
+from django.db.models import Q
+from rest_framework import status
+from rest_framework.response import Response
+
+from slm.api.edit import views as slm_views
+from slm.map.api.edit.serializers import StationMapSerializer, StationSerializer
+
+
+class StationListViewSet(slm_views.StationListViewSet):
+    serializer_class = StationSerializer
+
+    ordering_fields = (
+        slm_views.StationListViewSet.ordering_fields
+    )  # + ['latitude', 'longitude']
+
+    def get_queryset(self):
+        return super().get_queryset().with_location_fields("llh")
+
+
+# todo we could use geodjango and gis drf extensions to do this automatically - but that produces a large dependency
+#   overhead for a pretty basic task - revisit this if polygonal queries are deemed useful or other reasons to integrate
+#   GIS features arise
+class StationMapViewSet(StationListViewSet):
+    """
+    A view for returning a site list as a geojson set of point features. We
+    inherit from our normal StationListViewSet so all filtering parameters are
+    the same and we can pair requests to the map view with requests to the
+    station list view.
+    """
+
+    serializer_class = StationMapSerializer
+    pagination_class = None
+
+    def list(self, request, **kwargs):
+        return Response(
+            {
+                "type": "FeatureCollection",
+                "features": self.get_serializer(
+                    self.filter_queryset(self.get_queryset()), many=True
+                ).data,
+            },
+            status=status.HTTP_200_OK,
+        )
+
+    def get_queryset(self):
+        return super().get_queryset().filter(Q(llh__isnull=False))

slm/map/api/public/serializers.py

@@ -0,0 +1,29 @@
+from rest_framework import serializers
+
+from slm.api.public import serializers as slm_serializers
+
+
+class StationListSerializer(slm_serializers.StationListSerializer):
+    class Meta(slm_serializers.StationListSerializer.Meta):
+        # could add additional fields here to extend the data exposed by this api
+        fields = slm_serializers.StationListSerializer.Meta.fields
+
+
+class StationMapSerializer(serializers.Serializer):
+    def to_representation(self, instance):
+        return {
+            "type": "Feature",
+            "geometry": {
+                "type": "Point",
+                "coordinates": (
+                    [instance.llh[1], instance.llh[0]] if instance.llh else [None, None]
+                ),
+            },
+            "properties": {
+                "name": instance.name,
+                "status": instance.status,
+                "last_data": (
+                    max(0, instance.last_data.days) if instance.last_data else None
+                ),
+            },
+        }

slm/map/api/public/views.py

@@ -0,0 +1,64 @@
+from rest_framework import pagination
+from rest_framework.response import Response
+
+from slm.api.public import views as slm_views
+from slm.map.api.public.serializers import StationListSerializer, StationMapSerializer
+from slm.models import Site
+
+
+class StationListViewSet(slm_views.StationListViewSet):
+    serializer_class = StationListSerializer
+
+    # could add fields here to extend ordering options
+    ordering_fields = slm_views.StationListViewSet.ordering_fields
+
+    def get_queryset(self):
+        # could add additional filter parameters or annotations here to extend the api
+        return super().get_queryset()
+
+
+class FeatureCollectionPagination(pagination.BasePagination):
+    """
+    We implement geojson wrapper as a special type of pagination even though
+    no paging is done. This method fits neatly into how the base classes
+    generate and process response objects. Doing this directly in list()
+    on the view breaks filter form rendering in the browsable renderers.
+    """
+
+    def paginate_queryset(self, queryset, request, view=None):
+        return queryset
+
+    def to_html(self):
+        return None
+
+    def get_results(self, data):
+        return data["features"]
+
+    def get_paginated_response(self, data):
+        return Response({"type": "FeatureCollection", "features": data})
+
+    def get_paginated_response_schema(self, schema):
+        return {
+            "type": "object",
+            "properties": {
+                "type": {
+                    "type": "string",
+                    "example": "FeatureCollection",
+                },
+                "features": schema,
+            },
+        }
+
+
+class StationMapViewSet(StationListViewSet):
+    """
+    A view for returning a site list as a geojson set of point features.
+    """
+
+    serializer_class = StationMapSerializer
+    pagination_class = FeatureCollectionPagination
+
+    def get_queryset(self):
+        return (
+            Site.objects.with_location_fields("llh").public().availability().distinct()
+        )

slm/map/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class MapConfig(AppConfig):
+    name = "slm.map"
+    label = name.replace(".", "_")
+    verbose_name = " SLM Map"

slm/map/defines.py

@@ -0,0 +1,53 @@
+from django.utils.translation import gettext_lazy as _
+from django_enum import IntegerChoices
+from enum_properties import p, s
+
+
+class MapBoxStyle(IntegerChoices, s("slug", case_fold=True), p("version")):
+    """
+    https://docs.mapbox.com/api/maps/styles/
+    """
+
+    _symmetric_builtins_ = ["name", s("label", case_fold=True), "uri"]
+
+    # name             value    label                slug           version
+    STREETS = 1, _("Streets"), "streets", 12
+    OUTDOORS = 2, _("Outdoors"), "outdoors", 12
+    LIGHT = 3, _("Light"), "light", 11
+    DARK = 4, _("Dark"), "dark", 11
+    SATELLITE = 5, _("Satellite"), "satellite", 9
+    SATELLITE_STREETS = 6, _("Satellite Streets"), "satellite-streets", 12
+    NAVIGATION_DAY = 7, _("Navigation Day"), "navigation-day", 1
+    NAVIGATION_NIGHT = 8, _("Navigation Night"), "navigation-night", 1
+
+    @property
+    def uri(self):
+        return f"mapbox://styles/mapbox/{self.version_slug}"
+
+    @property
+    def version_slug(self):
+        return f"{self.slug}-v{self.version}"
+
+    def __str__(self):
+        return self.uri
+
+
+class MapBoxProjection(IntegerChoices, s("slug", case_fold=True)):
+    """
+    https://docs.mapbox.com/mapbox-gl-js/style-spec/projection/
+    """
+
+    _symmetric_builtins_ = ["name", "label"]
+
+    # name                  value   label                        slug
+    ALBERS = 0, _("Albers"), "albers"
+    EQUAL_EARTH = 1, _("Equal Earth"), "equalEarth"
+    EQUI_RECTANGULAR = 2, _("Equi-Rectangular"), "equirectangular"
+    LAMBERT_CONFORMAL_CONIC = 3, _("Lambert Conformal Conic"), "lambertConformalConic"
+    MERCATOR = 4, _("Mercator"), "mercator"
+    NATURAL_EARTH = 5, _("Natural Earth"), "naturalEarth"
+    WINKEL_TRIPEL = 6, _("Winkel Tripel"), "winkelTripel"
+    GLOBE = 7, _("Globe"), "globe"
+
+    def __str__(self):
+        return self.slug