igs-slm 0.1.0b0__py3-none-any.whl

slm/api/fields.py

@@ -0,0 +1,89 @@
+from datetime import date, datetime, time, timezone
+
+from dateutil import parser
+from django.conf import settings
+from django.contrib.gis.geos import Point
+from django.utils.translation import gettext as _
+from rest_framework.serializers import DateTimeField, Field
+
+
+class SLMDateTimeField(DateTimeField):
+    default_error_messages = {
+        "invalid": _("Unable to interpret datetime, please use format: {format}."),
+        "parse": _("Please use format: {format}: {error}"),
+    }
+
+    """
+    A much more lenient datetime field that uses dateutil to parse. This field
+    differs from the vanilla DRF DateTimeField in several ways:
+    
+    1) dateutil.parser is used to parse incoming strings. This is very lenient.
+    2) Values that are just dates default to default_time if it is set, and
+        fail otherwise.
+    3) The timezone is set to UTC unless otherwise given.
+    
+    :param default_time: Use this time for incoming values that are just dates.
+        defaults to midnight.
+    :param default_timezone: This is the output timezone - defaults to UTC.
+    :param kwargs: kwargs for DRF base classes.
+    """
+
+    default_time = time(hour=0, minute=0, second=0)
+
+    def __init__(
+        self, default_time=default_time, default_timezone=timezone.utc, **kwargs
+    ):
+        self.default_time = default_time
+        super().__init__(default_timezone=default_timezone, **kwargs)
+
+    def default_timezone(self):
+        return timezone.utc if settings.USE_TZ else None
+
+    def to_internal_value(self, value):
+        if isinstance(value, date) and not isinstance(value, datetime):
+            # assume midnight
+            if not self.default_time:
+                self.fail("date")
+            value = datetime.combine(value, self.default_time)
+
+        if isinstance(value, datetime):
+            return self.enforce_timezone(value)
+
+        try:
+            parsed = parser.parse(value)
+            if parsed is not None:
+                return self.enforce_timezone(parsed)
+        except parser.ParserError as pe:
+            self.fail("parse", format="CCYY-MM-DDThh:mmZ", error=str(pe))
+        except (ValueError, TypeError):
+            pass
+
+        self.fail("invalid", format="CCYY-MM-DDThh:mmZ")
+
+
+class SLMPointField(Field):
+    default_error_messages = {
+        "invalid": _(
+            "Unable to interpret point ({data}), please use format: {format}."
+        ),
+        "missing": _("Must provide all {num} values, received: {received}."),
+    }
+
+    def to_representation(self, value):
+        return value.coords
+
+    def to_internal_value(self, data):
+        try:
+            coords = [None if coord in ["", None] else float(coord) for coord in data]
+            # all values must be null or a number
+            if any([coord is None for coord in coords]) and not all(
+                [coord is None for coord in coords]
+            ):
+                self.fail(
+                    "missing",
+                    num=len(coords),
+                    received=len([coord for coord in coords if coord is not None]),
+                )
+            return Point(*coords) or None
+        except (TypeError, ValueError):
+            self.fail("invalid", format="[float, float, float]", data=data)

slm/api/filter.py

@@ -0,0 +1,504 @@
+from collections import namedtuple
+
+import django_filters
+from dateutil import parser
+from django.core.exceptions import ValidationError
+from django.db.models import Q
+from django.forms import DateTimeField
+from django.forms.utils import from_current_timezone
+from django.http import QueryDict
+from django.utils.functional import cached_property
+from django.utils.timezone import now
+from django.utils.translation import gettext as _
+from django_filters import (
+    BaseInFilter,
+    BooleanFilter,
+    DateTimeFilter,
+    FilterSet,
+    NumberFilter,
+    compat,
+)
+from django_filters.filterset import BaseFilterSet
+from rest_framework.filters import SearchFilter
+
+from slm.defines import AlertLevel, FrequencyStandardType, ISOCountry, SiteLogStatus
+from slm.forms import SLMBooleanField
+from slm.models import (
+    Agency,
+    Alert,
+    Antenna,
+    Network,
+    Radome,
+    Receiver,
+    SatelliteSystem,
+    Site,
+    SiteAntenna,
+    SiteIdentification,
+    SiteMoreInformation,
+    SiteReceiver,
+)
+
+
+class SiteSearchFilter(SearchFilter):
+    """
+    For large data textual search will get very slow. The database indexes on
+    these fields are not utilized for icontains lookups. PostgreSQL has a
+    built in textual search index that can be utilized with __search lookups
+    in Django. Alternatively an external solution would have to be implemented
+    for mysql.
+
+    On the current size of the IGS data, icontains does not introduce too large
+    of a performance penalty.
+    """
+
+    def filter_queryset(self, request, queryset, view):
+        """
+        This search was broken out manually because the query generated by
+        the ORM using the default DRF SearchFilter did not perform well enough.
+
+        The basic strategy is to do table text searches separately then pull
+        the relevant primary keys for matching sites into a set that the main
+        query with annotations and ordering simply requires a pk lookup. The
+        order_by() statements below remove unnecessary default ORDER BY
+        statements.
+
+        This strategy could be sped up more by doing the individual table
+        searches asynchronously.
+        """
+        terms = [
+            term.lstrip(",").rstrip(",").strip()
+            for term in self.get_search_terms(request)
+        ]
+        searched = Q()
+        if terms:
+            try:
+                epoch = parser.parse(request.GET.get("epoch", None)) or now()
+            except (parser.ParserError, TypeError, ValueError):
+                epoch = now()
+
+            for search_term in terms:
+                site_ids = set(
+                    Site.objects.filter(name__icontains=search_term)
+                    .order_by()
+                    .values_list("pk", flat=True)
+                )
+
+                site_ids.update(
+                    Site.objects.filter(
+                        networks__in=Network.objects.filter(name__icontains=search_term)
+                    )
+                    .order_by()
+                    .distinct()
+                    .values_list("pk", flat=True)
+                )
+
+                site_ids.update(
+                    Site.objects.filter(
+                        agencies__in=Agency.objects.filter(
+                            Q(name__icontains=search_term)
+                            | Q(shortname__icontains=search_term)
+                        )
+                    )
+                    .order_by()
+                    .distinct()
+                    .values_list("pk", flat=True)
+                )
+
+                site_receivers = SiteReceiver.objects.filter(
+                    Q(published=True)
+                    & (
+                        Q(
+                            receiver_type__in=Receiver.objects.filter(
+                                model__icontains=search_term
+                            )
+                        )
+                        | Q(firmware__icontains=search_term)
+                        | Q(serial_number__icontains=search_term)
+                        & (
+                            (Q(installed__lte=epoch) | Q(installed__isnull=True))
+                            & (Q(removed__gt=epoch) | Q(removed__isnull=True))
+                        )
+                    )
+                ).order_by()
+                site_antennas = SiteAntenna.objects.filter(
+                    Q(published=True)
+                    & (
+                        Q(
+                            antenna_type__in=Antenna.objects.filter(
+                                model__icontains=search_term
+                            )
+                        )
+                        | Q(
+                            radome_type__in=Radome.objects.filter(
+                                model__icontains=search_term
+                            )
+                        )
+                        & (
+                            (Q(installed__lte=epoch) | Q(installed__isnull=True))
+                            & (Q(removed__gt=epoch) | Q(removed__isnull=True))
+                        )
+                    )
+                ).order_by()
+
+                site_ids.update(
+                    Site.objects.filter(siteantenna__in=site_antennas)
+                    .order_by()
+                    .distinct()
+                    .values_list("pk", flat=True)
+                )
+
+                site_ids.update(
+                    Site.objects.filter(
+                        siteidentification__in=SiteIdentification.objects.filter(
+                            Q(published=True)
+                            & Q(iers_domes_number__icontains=search_term)
+                        ).order_by()
+                    )
+                    .order_by()
+                    .values_list("pk", flat=True)
+                )
+
+                site_ids.update(
+                    Site.objects.filter(
+                        sitemoreinformation__in=SiteMoreInformation.objects.filter(
+                            Q(published=True) & Q(primary__icontains=search_term)
+                        ).order_by()
+                    )
+                    .order_by()
+                    .values_list("pk", flat=True)
+                )
+
+                site_ids.update(
+                    Site.objects.filter(sitereceiver__in=site_receivers)
+                    .order_by()
+                    .distinct()
+                    .values_list("pk", flat=True)
+                )
+
+                searched &= Q(pk__in=site_ids)
+
+        return queryset.filter(searched)
+
+
+class SLMBooleanFilter(BooleanFilter):
+    field_class = SLMBooleanField
+
+
+class CrispyFormCompat:
+    """
+    Ensure the given form as a submit button and correct method set!
+    """
+
+    @property
+    def form(self):
+        form = BaseFilterSet.form.fget(self)
+
+        if compat.is_crispy():
+            from crispy_forms.helper import FormHelper
+            from crispy_forms.layout import Layout, Submit
+
+            helper = getattr(form, "helper", None)
+            if helper:
+                # add on a submit button if one does not exist on the form
+                def has_submit(fields):
+                    for field in fields:
+                        if isinstance(field, Submit) or has_submit(
+                            getattr(field, "fields", [])
+                        ):
+                            return True
+                    return False
+
+                if not has_submit(form.helper.layout.fields):
+                    helper.layout = Layout(
+                        *helper.layout.fields,
+                        Submit("", _("Submit"), css_class="btn btn-primary"),
+                    )
+            else:
+                form.helper = FormHelper()
+                form.helper.form_method = "GET"
+                form.helper.layout = Layout(
+                    *form.fields.keys(),
+                    Submit("", _("Submit"), css_class="btn btn-primary"),
+                )
+        return form
+
+
+class AcceptListArguments:
+    """
+    Automatic conversion of lists to GET parameters in AJAX seems to add pesky
+    [] to the end of list arguments - there doesn't seem to be an easy way
+    to handle this in FilterSets so we strip the brackets out with this mixin.
+
+    It seems really really stupid that this is necessary...
+    """
+
+    def __init__(self, data=None, *args, **kwargs):
+        if data:
+            stripped = QueryDict(mutable=True)
+            for key in data.keys():
+                if key.endswith("[]"):
+                    stripped.setlist(key[0:-2], data.getlist(key))
+                else:
+                    stripped.setlist(key, data.getlist(key))
+                    if len(stripped.getlist(key)) == 1:
+                        stripped[key] = stripped.getlist(key)[0]
+            data = stripped
+        super().__init__(data=data, *args, **kwargs)
+
+
+class MustIncludeThese(BaseInFilter, NumberFilter):
+    def __init__(self, field_name="pk", *args, **kwargs):
+        super().__init__(field_name=field_name, *args, **kwargs)
+
+    def filter(self, qs, value):
+        if value:
+            qs |= super().filter(qs.model.objects.all(), value).distinct()
+        return qs
+
+
+class SLMDateTimeField(DateTimeField):
+    """
+    A DateTimeField that uses dateutil to parse datetimes. Much more lenient
+    than the default parsers.
+    """
+
+    def to_python(self, value):
+        try:
+            return super().to_python(value)
+        except ValidationError:
+            try:
+                return from_current_timezone(parser.parse(value))
+            except parser.ParserError as pe:
+                raise ValidationError(
+                    _("Invalid date/time: {error}").format(str(pe)), code="invalid"
+                )
+
+
+class SLMDateTimeFilter(DateTimeFilter):
+    field_class = SLMDateTimeField
+
+
+class InitialValueFilterSet(FilterSet):
+    """
+    This allows you to specify initial values that can also be callables on
+    your field filters.
+    """
+
+    def __init__(self, data=None, *args, **kwargs):
+        # if filterset is bound, use initial values as defaults
+        if data is not None:
+            # get a mutable copy of the QueryDict
+            data = data.copy()
+            for name, f in self.base_filters.items():
+                initial = f.extra.get("initial")
+                # filter param is either missing or empty, use initial as
+                # default
+                if not data.get(name) and initial:
+                    data[name] = initial() if callable(initial) else initial
+
+        super().__init__(data, *args, **kwargs)
+
+
+class BaseStationFilter(CrispyFormCompat, AcceptListArguments, FilterSet):
+    EQLookup = namedtuple("EQLookup", "relation field start end")
+    EQUIPMENT_TABLE = {
+        "satellite_system": EQLookup(
+            "sitereceiver", "satellite_system", "installed", "removed"
+        ),
+        "receiver": EQLookup("sitereceiver", "receiver_type", "installed", "removed"),
+        "antenna": EQLookup("siteantenna", "antenna_type", "installed", "removed"),
+        "radome": EQLookup("siteantenna", "radome_type", "installed", "removed"),
+        "frequency_standard": EQLookup(
+            "sitefrequencystandard", "standard_type", "effective_start", "effective_end"
+        ),
+    }
+
+    SITE_FILTER = Site.objects.all()
+
+    @property
+    def restrict_published(self):
+        return False
+
+    def published_q(self, lookup):
+        if self.restrict_published:
+            if lookup:
+                lookup += "__"
+            return Q(**{f"{lookup}published": True})
+        return Q()
+
+    @property
+    def current_equipment(self):
+        return self.form.cleaned_data.get("current", None)
+
+    @cached_property
+    def alert_fields(self):
+        """
+        Fetch the mapping of alert names to related fields.
+        """
+
+        def get_related_field(alert):
+            for obj in Site._meta.related_objects:
+                if obj.related_model is alert:
+                    return obj.name
+
+        return {
+            alert.__name__.lower(): get_related_field(alert)
+            for alert in Alert.objects.site_alerts()
+        }
+
+    @property
+    def query_epoch(self):
+        return self.form.cleaned_data.get("epoch", now())
+
+    epoch = django_filters.DateTimeFilter(field_name="epoch", method="at_epoch")
+
+    def at_epoch(self, queryset, name, value):
+        return queryset.at_epoch(epoch=value)
+
+    name = django_filters.CharFilter(
+        field_name="name", lookup_expr="icontains", distinct=True
+    )
+
+    station = django_filters.ModelMultipleChoiceFilter(
+        field_name="name",
+        to_field_name="name",
+        queryset=SITE_FILTER,
+        # method='filter_stations',
+        null_value="",
+        null_label="",
+    )
+
+    id = MustIncludeThese()
+
+    status = django_filters.MultipleChoiceFilter(
+        choices=SiteLogStatus.choices, distinct=True
+    )
+
+    alert = django_filters.MultipleChoiceFilter(
+        choices=[
+            (alert.__name__, alert._meta.verbose_name.title())
+            for alert in Alert.objects.site_alerts()
+        ],
+        method="filter_alerts",
+        distinct=True,
+    )
+
+    alert_level = django_filters.MultipleChoiceFilter(
+        choices=[(level.value, level.label) for level in AlertLevel],
+        method="filter_alert_level",
+    )
+
+    agency = django_filters.ModelMultipleChoiceFilter(
+        field_name="agencies", queryset=Agency.objects.all(), distinct=True
+    )
+
+    network = django_filters.ModelMultipleChoiceFilter(
+        field_name="networks", queryset=Network.objects.all(), distinct=True
+    )
+
+    receiver = django_filters.ModelMultipleChoiceFilter(
+        method="filter_equipment", queryset=Receiver.objects.all(), distinct=True
+    )
+
+    antenna = django_filters.ModelMultipleChoiceFilter(
+        method="filter_equipment", queryset=Antenna.objects.all(), distinct=True
+    )
+
+    radome = django_filters.ModelMultipleChoiceFilter(
+        method="filter_equipment", queryset=Radome.objects.all(), distinct=True
+    )
+
+    satellite_system = django_filters.ModelMultipleChoiceFilter(
+        method="filter_equipment", queryset=SatelliteSystem.objects.all()
+    )
+
+    frequency_standard = django_filters.MultipleChoiceFilter(
+        choices=FrequencyStandardType.choices, method="filter_equipment"
+    )
+
+    country = django_filters.MultipleChoiceFilter(
+        choices=ISOCountry.choices, method="filter_country"
+    )
+
+    current = SLMBooleanFilter(method="noop", distinct=True, field_name="current")
+
+    geography = django_filters.CharFilter(method="filter_geography")
+
+    def filter_geography(self, queryset, name, value):
+        qry = Q()
+        for poly in value:
+            qry |= Q(**{"sitelocation__llh__within": poly})
+        qry &= Q(sitelocation__published=True)
+        return queryset.filter(qry)
+
+    def noop(self, queryset, name, value):
+        return queryset
+
+    def filter_country(self, queryset, name, value):
+        return queryset.filter(
+            Q(sitelocation__country__in=value) & Q(self.published_q("sitelocation"))
+        )
+
+    def filter_alerts(self, queryset, name, value):
+        if value:
+            alert_q = Q()
+            for alert in value:
+                alert_q |= Q(**{f"{self.alert_fields[alert.lower()]}__isnull": False})
+            return queryset.filter(alert_q)
+        return queryset
+
+    def filter_alert_level(self, queryset, name, value):
+        if value:
+            level_q = Q()
+            for alerts in Site.alert_fields:
+                level_q |= Q(**{f"{alerts}__level__in": value})
+            return queryset.filter(level_q)
+        return queryset
+
+    def filter_equipment(self, queryset, name, value):
+        if value:
+            lookup = self.EQUIPMENT_TABLE[name]
+            if self.current_equipment:
+                return queryset.filter(
+                    Q(**{f"{lookup.relation}__{lookup.field}__in": value})
+                    & self.published_q(lookup.relation)
+                    & (
+                        Q(
+                            **{
+                                f"{lookup.relation}__{lookup.start}__lte": self.query_epoch
+                            }
+                        )
+                        | Q(**{f"{lookup.relation}__{lookup.start}__isnull": True})
+                    )
+                    & (
+                        Q(**{f"{lookup.relation}__{lookup.end}__gt": self.query_epoch})
+                        | Q(**{f"{lookup.relation}__{lookup.end}__isnull": True})
+                    )
+                ).distinct()
+            else:
+                return queryset.filter(
+                    Q(**{f"{lookup.relation}__{lookup.field}__in": value})
+                    & self.published_q(lookup.relation)
+                ).distinct()
+        return queryset
+
+    class Meta:
+        model = Site
+        fields = (
+            "id",
+            "status",
+            "alert_level",
+            "station",
+            "name",
+            "agency",
+            "network",
+            "receiver",
+            "antenna",
+            "radome",
+            "country",
+            "current",
+            "epoch",
+            "satellite_system",
+            "frequency_standard",
+        )
+        distinct = True

slm/api/pagination.py

@@ -0,0 +1,55 @@
+from rest_framework.pagination import LimitOffsetPagination
+from rest_framework.response import Response
+
+
+class BrowsablePagination(LimitOffsetPagination):
+    default_limit = 100
+
+
+class ClientPagination(LimitOffsetPagination):
+    default_limit = 10
+
+
+class DataTablesPagination(LimitOffsetPagination):
+    default_limit = 20
+
+    # datatables naming
+    limit_query_param = "length"
+    offset_query_param = "start"
+
+    queryset = None
+    draw = None
+
+    def paginate_queryset(self, queryset, request, view=None):
+        self.queryset = queryset
+        self.draw = request.query_params.get("draw", None)
+        return super().paginate_queryset(queryset, request, view=view)
+
+    def get_paginated_response(self, data):
+        resp = {
+            "data": data,
+            "recordsTotal": self.queryset.model.objects.count(),
+            "recordsFiltered": self.count,
+            "next": self.get_next_link(),
+            "previous": self.get_previous_link(),
+        }
+        if self.draw is not None:
+            resp["draw"] = self.draw
+        return Response(resp)
+
+    def get_paginated_response_schema(self, schema):
+        return {
+            "type": "object",
+            "properties": {
+                "recordsTotal": {
+                    "type": "integer",
+                    "example": 123,
+                },
+                "recordsFiltered": {"type": "integer", "example": 123},
+                "draw": {"type": "integer", "nullable": True, "example": 1},
+                "data": schema,
+            },
+        }
+
+    def get_results(self, data):
+        return data["data"]