igs-slm 0.1.2b0__py3-none-any.whl → 0.1.5b0__py3-none-any.whl

slm/models/system.py

@@ -1,7 +1,9 @@
 import os
+import typing as t
 from datetime import datetime, timezone
 from io import BytesIO
 from logging import getLogger
+from pathlib import Path
 
 from dateutil import parser
 from django.conf import settings
@@ -14,6 +16,8 @@ from django.urls import reverse
 from django.utils.timezone import is_naive, make_aware, now
 from django.utils.translation import gettext as _
 from django_enum import EnumField
+from packaging.version import Version
+from packaging.version import parse as parse_version
 from PIL import Image
 from polymorphic.managers import PolymorphicManager, PolymorphicQuerySet
 from polymorphic.models import PolymorphicModel
@@ -27,9 +31,46 @@ from slm.defines import (
     SLMFileType,
 )
 from slm.models.sitelog import DefaultToStrEncoder
+from slm.singleton import SingletonModel
 from slm.utils import get_exif_tags
 
 
+def site_upload_path(instance: "SiteFile", filename: str) -> str:
+    """
+    file will be saved to:
+        MEDIA_ROOT/uploads/<site name>/filename
+
+    :param filename: The name of the file
+    :return: The path where the site file should reside.
+    """
+    from .index import ArchivedSiteLog
+
+    prefix = Path()
+    if instance.SUB_DIRECTORY:
+        prefix = Path(instance.SUB_DIRECTORY)
+    dest = prefix / instance.site.name / filename
+    timestamp = (
+        instance.index.begin
+        if isinstance(instance, ArchivedSiteLog)
+        else instance.timestamp
+    )
+    if (Path(settings.MEDIA_ROOT) / dest).exists():
+        stem, suffix = dest.stem, dest.suffix
+        dest = dest.with_name(f"{stem}_{timestamp.strftime('%H%M%S')}{suffix}")
+    return dest.as_posix()
+
+
+def site_thumbnail_path(instance: "SiteFile", filename: str) -> str:
+    """
+    Return the path for the thumbnail image for the given filename.
+
+    :param filename: The name of the file
+    :return: The path where the thumbnail image should reside.
+    """
+    path = Path(instance.upload_path(filename))
+    return (path.parent / "thumbnails" / path.name).as_posix()
+
+
 class AgencyManager(models.Manager):
     pass
 
@@ -144,33 +185,6 @@ class Network(models.Model):
         return self.name
 
 
-def site_upload_path(instance, filename):
-    """
-     file will be saved to:
-        MEDIA_ROOT/uploads/<site name>/filename
-
-    :param instance: The SiteFile instance
-    :param filename: The name of the file
-    :return: The path where the site file should reside.
-    """
-    prefix = ""
-    if instance.SUB_DIRECTORY:
-        prefix = f"{instance.SUB_DIRECTORY}/"
-    return f"{prefix}{instance.site.name}/{filename}"
-
-
-def site_thumbnail_path(instance, filename):
-    """
-    Return the path for the thumbnail image for the given filename.
-
-    :param instance: The SiteFile instance
-    :param filename: The name of the file
-    :return: The path where the thumbnail image should reside.
-    """
-    parts = str(site_upload_path(instance, filename)).split("/")
-    return "/".join([*parts[0:-1], "thumbnails", parts[-1]])
-
-
 class SiteFile(models.Model):
     SUB_DIRECTORY = "misc"
 
@@ -185,7 +199,9 @@ class SiteFile(models.Model):
     )
 
     timestamp = models.DateTimeField(
-        auto_now_add=True, db_index=True, help_text=_("When the file was uploaded.")
+        auto_now_add=True,
+        db_index=True,
+        help_text=_("When the file was created or uploaded."),
     )
 
     file = models.FileField(
@@ -193,6 +209,7 @@ class SiteFile(models.Model):
         null=False,
         max_length=255,
         help_text=_("A pointer to the uploaded file on disk."),
+        unique=True,
     )
 
     size = models.PositiveIntegerField(null=True, default=None, blank=True)
@@ -480,7 +497,7 @@ class SiteFileUpload(SiteFile):
         blank=True,
         db_index=True,
         help_text=_(
-            "The status of the file. This will also depend on what type the " "file is."
+            "The status of the file. This will also depend on what type the file is."
         ),
     )
 
@@ -655,9 +672,9 @@ class LogEntry(PolymorphicModel):
     def __str__(self):
         return (
             f"({self.site.name} | "
-            f'{self.user.name or self.user.email if self.user else ""}) '
+            f"{self.user.name or self.user.email if self.user else ''}) "
             f"[{self.timestamp}]: {self.type} -> "
-            f'{self.section or self.file or self.site or ""}'
+            f"{self.section or self.file or self.site or ''}"
         )
 
     class Meta:
@@ -721,3 +738,27 @@ class SiteTideGauge(models.Model):
 
     class Meta:
         ordering = ("site", "distance")
+
+
+class SLMVersion(SingletonModel):
+    """
+    We store the SLM code version in the database to enable safe upgrades.
+    """
+
+    version_str = models.CharField(default="", blank=True)
+
+    @classmethod
+    def update(cls, version: t.Optional[Version] = None):
+        if not version:
+            from slm import __version__ as slm_version
+
+            version = parse_version(slm_version)
+        instance = cls.load()
+        instance.version_str = str(version)
+        instance.save()
+
+    @property
+    def version(self) -> t.Optional[Version]:
+        if self.version_str:
+            return parse_version(self.version_str)
+        return None

slm/models/user.py

@@ -285,7 +285,7 @@ class User(AbstractBaseUser, PermissionsMixin):
 
     @property
     def full_name(self):
-        return f'{self.first_name or ""} {self.last_name or ""}'.strip()
+        return f"{self.first_name or ''} {self.last_name or ''}".strip()
 
     def __str__(self):
         if self.name:

slm/parsing/__init__.py

@@ -1,4 +1,5 @@
 import re
+import typing as t
 from dataclasses import dataclass
 from datetime import date, datetime, timezone
 from functools import partial
@@ -59,7 +60,8 @@ def remove_from_start(value: str, prefixes: List[str]):
 
 @dataclass
 class _Ignored:
-    msg: str
+    msg: str = ""
+    columns: t.Optional[t.Tuple[int, int]] = None
 
 
 @dataclass
@@ -86,10 +88,24 @@ class Finding:
     A base class for parser/binding findings.
     """
 
+    lineno: int
+    parser: "BaseParser"
+    message: str
+    section: t.Optional["BaseSection"]
+    parameter: t.Optional["BaseParameter"]
+    line: t.Optional[str]
     priority: int = 0
+    columns: t.Optional[t.Tuple[int, int]]
 
     def __init__(
-        self, lineno, parser, message, section=None, parameter=None, line=None
+        self,
+        lineno,
+        parser,
+        message,
+        section=None,
+        parameter=None,
+        line=None,
+        columns=None,
     ):
         self.lineno = lineno
         self.parser = parser
@@ -97,11 +113,12 @@ class Finding:
         self.section = section
         self.parameter = parameter
         self.line = line
+        self.columns = columns
 
     def __str__(self):
         return (
-            f"({self.lineno+1: 4}) {self.parser.lines[self.lineno]}"
-            f'{" " * (80-len(self.parser.lines[self.lineno]))}'
+            f"({self.lineno + 1: 4}) {self.parser.lines[self.lineno]}"
+            f"{' ' * (80 - len(self.parser.lines[self.lineno]))}"
             f"[{self.level.upper()}]: {self.message}"
         )
 
@@ -265,7 +282,7 @@ class BaseSection:
         section_str = f"{self.index_string} {self.header}\n"
         if self.example:
             section_str = f"{self.index_string} {self.header} (EXAMPLE)\n"
-        for name, param in self.parameters.items():
+        for param in self.parameters.values():
             section_str += f"\t{param}\n"
         return section_str
 
@@ -275,7 +292,7 @@ class BaseSection:
         True if any parameter in this section contains a real value that is
         not a placeholder.
         """
-        for name, param in self.parameters.items():
+        for param in self.parameters.values():
             if not (param.is_empty or param.is_placeholder):
                 return True
         return False
@@ -290,7 +307,7 @@ class BaseSection:
             else:
                 index += "x"
             if self.subsection_number and (self.order or self.example):
-                index += f'.{self.order if self.order else "x"}'
+                index += f".{self.order if self.order else 'x'}"
         return index
 
     @property
@@ -350,7 +367,7 @@ class BaseParser:
     @property
     def findings_context(self) -> Dict[int, Tuple[str, str]]:
         return {
-            int(line): (finding.level, finding.message)
+            int(line): (finding.level, finding.message, finding.columns)
             for line, finding in self.findings.items()
         }
 
@@ -476,7 +493,7 @@ def to_antenna(value):
     except Antenna.DoesNotExist:
         antennas = "\n".join([ant.model for ant in Antenna.objects.public()])
         raise ValueError(
-            f"Unexpected antenna model {antenna}. Must be one of " f"\n{antennas}"
+            f"Unexpected antenna model {antenna}. Must be one of \n{antennas}"
         )
 
 
@@ -487,7 +504,7 @@ def to_radome(value):
     except Radome.DoesNotExist:
         radomes = "\n".join([rad.model for rad in Radome.objects.public()])
         raise ValueError(
-            f"Unexpected radome model {radome}. Must be one of " f"\n{radomes}"
+            f"Unexpected radome model {radome}. Must be one of \n{radomes}"
         )
 
 
@@ -498,7 +515,7 @@ def to_receiver(value):
     except Receiver.DoesNotExist:
         receivers = "\n".join([rec.model for rec in Receiver.objects.public()])
         raise ValueError(
-            f"Unexpected receiver model {receiver}. Must be one of " f"\n{receivers}"
+            f"Unexpected receiver model {receiver}. Must be one of \n{receivers}"
         )
 
 
@@ -546,7 +563,7 @@ def to_enum(enum_cls, value, strict=False, blank=None, ignored=None):
             return value.strip()
 
         valid_list = "  \n".join(en.label for en in enum_cls)
-        raise ValueError(f"Invalid value {value} must be one of:\n" f"{valid_list}")
+        raise ValueError(f"Invalid value {value} must be one of:\n{valid_list}")
     return blank
 
 
@@ -665,14 +682,14 @@ def to_decimal_degrees(value):
     flt = to_float(value)
     if isinstance(flt, _Warning):
         return _Warning(msg=flt.msg, value=dddmmssss_to_decimal(flt.value))
-    if flt is _Ignored or isinstance(flt, _Ignored):
+    if isinstance(flt, _Ignored):
         return flt
     return dddmmssss_to_decimal(to_float(value))
 
 
 def to_int(value, units=None, prefixes=None):
     try:
-        to_numeric(numeric_type=int, value=value, units=units, prefixes=prefixes)
+        return to_numeric(numeric_type=int, value=value, units=units, prefixes=prefixes)
     except ValueError:
         from_float = to_numeric(
             numeric_type=float, value=value, units=units, prefixes=prefixes
@@ -718,12 +735,12 @@ def to_pressure(value):
 def _to_date(value):
     if value.strip():
         if "CCYY-MM-DD" in value.upper():
-            return _Ignored
+            return _Ignored(msg=f"{value} is a placeholder.")
         try:
             return parse_date(value, tzinfos=TZ_INFOS).date()
         except Exception as exc:
             raise ValueError(
-                f"Unable to parse {value} into a date. Expected " f"format: CCYY-MM-DD"
+                f"Unable to parse {value} into a date. Expected format: CCYY-MM-DD"
             ) from exc
     return None
 
@@ -731,7 +748,7 @@ def _to_date(value):
 def _to_datetime(value):
     if value.strip():
         if "CCYY-MM-DD" in value.upper():
-            return _Ignored
+            return _Ignored(msg=f"{value} is a placeholder.")
         try:
             # UT and UTUT has been seen as a timezone specifier in the wild
             dt = parse_date(value, tzinfos=TZ_INFOS)

slm/parsing/legacy/binding.py

@@ -71,7 +71,7 @@ def reg(name, header_index, bindings):
 def ignored(_, msg=""):
     if msg:
         return _Ignored(msg)
-    return _Ignored
+    return _Ignored()
 
 
 def to_temp_stab(value):
@@ -129,40 +129,33 @@ def to_temp_stab(value):
     return stabilized, nominal, deviation
 
 
-def effective_start(value):
+def effective_date(value: str, part: int, label: str):
     try:
-        start_str = ""
+        dt_str = ""
         if value.strip():
             sep = "/" if "/" in value else " - "
-            start_str = value.split(sep)[0]
-            return to_date(start_str)
+            splt = value.split(sep)
+            if len(splt) > part:
+                dt_str = splt[part]
+                if dt_str.strip():
+                    ret = to_date(dt_str.strip())
+                    if isinstance(ret, _Ignored):
+                        start = value.index(dt_str)
+                        end = start + len(dt_str)
+                        ret.columns = (start, end)
+                    return ret
         return None
     except ValueError as ve:
-        if start_str.upper() in DATE_PLACEHOLDERS:
+        if dt_str.upper() in DATE_PLACEHOLDERS:
             return None
         raise ValueError(
-            f"Unable to parse {value} into an expected start date. Expected "
+            f"Unable to parse {value} into an expected {label} date. Expected "
             f"format: CCYY-MM-DD/CCYY-MM-DD"
         ) from ve
 
 
-def effective_end(value):
-    try:
-        end_str = ""
-        if value.strip():
-            sep = "/" if "/" in value else " - "
-            splt = value.split(sep)
-            if len(splt) > 1:
-                end_str = value.split(sep)[1]
-                return to_date(end_str)
-        return None
-    except ValueError as ve:
-        if end_str.upper() in DATE_PLACEHOLDERS:
-            return None
-        raise ValueError(
-            f"Unable to parse {value} into an expected end date. Expected "
-            f"format: CCYY-MM-DD/CCYY-MM-DD"
-        ) from ve
+effective_start = partial(effective_date, part=0, label="start")
+effective_end = partial(effective_date, part=1, label="end")
 
 
 def no_sat_warning(line_no, parser, satellites):
@@ -462,12 +455,12 @@ class SiteLogBinder(BaseBinder):
                     ("Tied Marker Usage", ("usage", to_str)),
                     ("Tied Marker CDP Number", ("cdp_number", to_str)),
                     ("Tied Marker DOMES Number", ("domes_number", to_str)),
-                    ("dx", ("dx", to_float)),
-                    ("dy", ("dy", to_float)),
-                    ("dz", ("dz", to_float)),
-                    ("dx (m)", ("dx", to_float)),
-                    ("dy (m)", ("dy", to_float)),
-                    ("dz (m)", ("dz", to_float)),
+                    ("dx", ("dx", partial(to_float, units=["m"]))),
+                    ("dy", ("dy", partial(to_float, units=["m"]))),
+                    ("dz", ("dz", partial(to_float, units=["m"]))),
+                    ("dx (m)", ("dx", partial(to_float, units=["m"]))),
+                    ("dy (m)", ("dy", partial(to_float, units=["m"]))),
+                    ("dz (m)", ("dz", partial(to_float, units=["m"]))),
                     (
                         "Accuracy",
                         (
@@ -656,13 +649,43 @@ class SiteLogBinder(BaseBinder):
         },
     }
 
+    MULTIPLE_ENTRIES = {
+        3,
+        4,
+        5,
+        6,
+        7,
+        (8, 1),
+        (8, 2),
+        (8, 3),
+        (8, 4),
+        (8, 5),
+        (9, 1),
+        (9, 2),
+        (9, 3),
+        10,
+    }
+
     def __init__(self, parsed: SiteLogParser):
         super().__init__(parsed)
-        for _1, section in self.parsed.sections.items():
-            if section.example or not section.contains_values:
+        for section in self.parsed.sections.values():
+            if section.example:
+                continue
+            section_depth = sum(1 for item in section.index_tuple if item is not None)
+            heading_depth = (
+                1
+                if not isinstance(section.heading_index, tuple)
+                else len(section.heading_index)
+            )
+            is_header = (
+                section.heading_index in self.MULTIPLE_ENTRIES
+                and section_depth == heading_depth
+            )
+            if is_header and not section.contains_values:
+                # skip empty headers
                 continue
             if section.heading_index not in self.TRANSLATION_TABLE:
-                for line_no in range(section.line_no, section.line_end):
+                for _ in range(section.line_no, section.line_end):
                     self.parsed.add_finding(
                         Warn(
                             section.line_no,
@@ -738,16 +761,24 @@ class SiteLogBinder(BaseBinder):
                         if parameter.is_placeholder
                         else parse(parameter.value)
                     )
-                    if value == _Ignored or isinstance(value, _Ignored):
+                    if isinstance(value, _Ignored):
+                        cols = None
+                        if cols := getattr(value, "columns", None):
+                            val_start = self.lines[parameter.line_no].index(
+                                parameter.value
+                            )
+                            cols = (cols[0] + val_start, cols[1] + val_start)
                         self.parsed.add_finding(
                             Ignored(
                                 parameter.line_no,
                                 self.parsed,
                                 getattr(value, "msg", _("Parameter is ignored")),
                                 section=section,
+                                columns=cols,
                             )
                         )
                         ignored.add(param)
+                        parameter.bind(param, None)
                     elif isinstance(value, _Warning):
                         self.parsed.add_finding(
                             Warn(

slm/parsing/legacy/parser.py

@@ -52,8 +52,8 @@ class ParsedParameter(BaseParameter):
         super().__init__(
             line_no=line_no,
             name=(
-                f'{sub_heading if sub_heading else ""}'
-                f'{"::" if sub_heading else ""}'
+                f"{sub_heading if sub_heading else ''}"
+                f"{'::' if sub_heading else ''}"
                 f"{match.group(1).strip()}"
             ),
             values=[match.group(2).strip()],

slm/parsing/xsd/binding.py

@@ -36,7 +36,7 @@ class SiteLogBinder(BaseBinder):
     TRANSLATION_TABLE[GeodesyMLVersion.v0_5] = {
         (
             (0,),
-            "/geo:GeodesyML/geo:siteLog/geo:formInformation/" "geo:FormInformation",
+            "/geo:GeodesyML/geo:siteLog/geo:formInformation/geo:FormInformation",
         ): TRANSLATION_TABLE[GeodesyMLVersion.v0_4][
             ((0,), "/geo:GeodesyML/geo:siteLog/geo:formInformation")
         ],

slm/parsing/xsd/parser.py

@@ -99,8 +99,7 @@ class SiteLogParser(BaseParser):
                     Error(
                         self.doc.sourceline - 1,
                         self,
-                        f"Unsupported schema: "
-                        f"{self.doc.nsmap.get(self.doc.prefix)}",
+                        f"Unsupported schema: {self.doc.nsmap.get(self.doc.prefix)}",
                     )
                 )
 

slm/receivers/__init__.py

@@ -6,6 +6,6 @@ def register():
     if not _registered:
         # the order of these imports is important, index receivers must happen
         # before alert receivers
-        from slm.receivers import alerts, cleanup, event_loggers, index
+        from slm.receivers import alerts, cleanup, event_loggers, index, migration
 
-        _registered = event_loggers and cleanup and index and alerts
+        _registered = event_loggers and cleanup and index and alerts and migration

slm/receivers/index.py

@@ -5,7 +5,7 @@ from slm.defines import SiteLogStatus
 
 
 @receiver(slm_signals.site_status_changed)
-def index_site(sender, site, previous_status, new_status, **kwargs):
+def index_site(sender, site, previous_status, new_status, reverted=False, **kwargs):
     from slm.models import ArchiveIndex
 
     if site.last_publish and (
@@ -18,6 +18,7 @@ def index_site(sender, site, previous_status, new_status, **kwargs):
         and previous_status in SiteLogStatus.active_states()
         and previous_status is not SiteLogStatus.PUBLISHED
         and new_status is SiteLogStatus.PUBLISHED
+        and not reverted
     ):
         # catch an edge case where a section publish triggers a whole log publish
         # these signals/edit state diagram needs to be cleaned up. this code is

slm/receivers/migration.py

@@ -0,0 +1,21 @@
+"""
+We update our SLMVersion tag when migrate is run - which should happen everytime the
+SLM software is updated.
+"""
+
+from django.db.models.signals import post_migrate, pre_migrate
+from django.dispatch import receiver
+
+from slm.models import SLMVersion
+
+
+@receiver(pre_migrate)
+def check_safe_upgrade(**_):
+    from slm.management.commands.check_upgrade import Command as CheckUpgrade
+
+    CheckUpgrade()
+
+
+@receiver(post_migrate)
+def update_slm_version(**_):
+    SLMVersion.update()